You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/02/19 05:26:19 UTC

[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1904 - Failure

Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1904/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:27019/md/tq/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:27019/md/tq/collection1
	at __randomizedtesting.SeedInfo.seed([5CC03A27199D994:842A8DBA06C6B9A8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:503)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:168)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:146)
	at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:663)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:150)
	at java.net.SocketInputStream.read(SocketInputStream.java:121)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
	... 47 more




Build Log:
[...truncated 10593 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> 2208907 T5907 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 2208907 T5907 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /md/tq
   [junit4]   2> 2208913 T5907 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1392783091016
   [junit4]   2> 2208914 T5907 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 2208915 T5908 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2209017 T5907 oasc.ZkTestServer.run start zk server on port:49190
   [junit4]   2> 2209018 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2209023 T5914 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60f33e28 name:ZooKeeperConnection Watcher:127.0.0.1:49190 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2209024 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2209024 T5907 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2209037 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2209039 T5916 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9232509 name:ZooKeeperConnection Watcher:127.0.0.1:49190/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2209040 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2209040 T5907 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2209050 T5907 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2209053 T5907 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2209056 T5907 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2209065 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2209066 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2209078 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2209079 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2209183 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2209184 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2209187 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2209188 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2209191 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2209192 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2209199 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2209200 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2209203 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2209203 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 2209206 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2209207 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2209214 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2209215 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2209218 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2209219 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2209222 T5907 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2209222 T5907 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2209233 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2209235 T5918 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38b13f2d name:ZooKeeperConnection Watcher:127.0.0.1:49190/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2209236 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2209598 T5907 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2209603 T5907 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2209607 T5907 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:40243
   [junit4]   2> 2209610 T5907 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2209610 T5907 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2209611 T5907 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392783091348
   [junit4]   2> 2209611 T5907 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392783091348/'
   [junit4]   2> 2209652 T5907 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392783091348/solr.xml
   [junit4]   2> 2209753 T5907 oasc.CoreContainer.<init> New CoreContainer 733519714
   [junit4]   2> 2209753 T5907 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392783091348/]
   [junit4]   2> 2209755 T5907 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2209756 T5907 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2209756 T5907 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2209757 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2209757 T5907 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2209758 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2209758 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2209759 T5907 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2209759 T5907 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2209775 T5907 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2209776 T5907 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2209776 T5907 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2209777 T5907 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49190/solr
   [junit4]   2> 2209777 T5907 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2209779 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2209785 T5929 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ebd0dcb name:ZooKeeperConnection Watcher:127.0.0.1:49190 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2209787 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2209790 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2209832 T5931 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e0fb56a name:ZooKeeperConnection Watcher:127.0.0.1:49190/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2209832 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2209836 T5907 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2209892 T5907 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2209936 T5907 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2209938 T5907 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40243_md%2Ftq
   [junit4]   2> 2209940 T5907 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40243_md%2Ftq
   [junit4]   2> 2209955 T5907 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 2209958 T5907 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2209969 T5907 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:40243_md%2Ftq
   [junit4]   2> 2209970 T5907 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2209977 T5907 oasc.Overseer.start Overseer (id=91277432653283332-127.0.0.1:40243_md%2Ftq-n_0000000000) starting
   [junit4]   2> 2209981 T5907 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2209994 T5933 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2209995 T5907 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2209995 T5933 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2209998 T5907 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2210000 T5907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2210002 T5932 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2210008 T5934 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2210008 T5934 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2210010 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2210010 T5934 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2210011 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2210012 T5932 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:40243/md/tq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40243_md%2Ftq",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2210012 T5932 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 2210012 T5932 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2210016 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2210021 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2211012 T5934 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2211012 T5934 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392783091348/collection1
   [junit4]   2> 2211012 T5934 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 2211013 T5934 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2211013 T5934 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2211014 T5934 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2211015 T5934 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392783091348/collection1/'
   [junit4]   2> 2211016 T5934 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392783091348/collection1/lib/classes/' to classloader
   [junit4]   2> 2211017 T5934 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392783091348/collection1/lib/README' to classloader
   [junit4]   2> 2211084 T5934 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 2211158 T5934 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2211260 T5934 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2211277 T5934 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2211951 T5934 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2211952 T5934 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2211953 T5934 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2211965 T5934 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2211970 T5934 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2211983 T5934 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2211988 T5934 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2211993 T5934 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2211994 T5934 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2211994 T5934 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2211995 T5934 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2211996 T5934 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2211996 T5934 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2211996 T5934 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2211997 T5934 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392783091348/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/control/data/
   [junit4]   2> 2211997 T5934 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4533a8de
   [junit4]   2> 2211998 T5934 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/control/data
   [junit4]   2> 2211998 T5934 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/control/data/index/
   [junit4]   2> 2211999 T5934 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2211999 T5934 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/control/data/index
   [junit4]   2> 2212000 T5934 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=59.2021484375, floorSegmentMB=1.990234375, forceMergeDeletesPctAllowed=18.048608453786404, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7408586669501135
   [junit4]   2> 2212001 T5934 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@60a083cb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5fbc25e3),segFN=segments_1,generation=1}
   [junit4]   2> 2212001 T5934 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2212005 T5934 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2212006 T5934 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2212006 T5934 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2212007 T5934 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2212007 T5934 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2212007 T5934 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2212008 T5934 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2212009 T5934 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2212009 T5934 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2212009 T5934 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2212010 T5934 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2212010 T5934 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2212011 T5934 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2212011 T5934 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2212012 T5934 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2212012 T5934 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2212029 T5934 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2212035 T5934 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2212035 T5934 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2212036 T5934 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=12.306640625, floorSegmentMB=0.96484375, forceMergeDeletesPctAllowed=1.5504112431909334, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.27580223564836803
   [junit4]   2> 2212037 T5934 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@60a083cb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5fbc25e3),segFN=segments_1,generation=1}
   [junit4]   2> 2212037 T5934 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2212038 T5934 oass.SolrIndexSearcher.<init> Opening Searcher@6d4dd227[collection1] main
   [junit4]   2> 2212041 T5935 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6d4dd227[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2212044 T5934 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2212044 T5907 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 2212045 T5907 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2212044 T5938 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:40243/md/tq collection:control_collection shard:shard1
   [junit4]   2> 2212047 T5938 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 2212051 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2212066 T5940 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e0b3177 name:ZooKeeperConnection Watcher:127.0.0.1:49190/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2212067 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2212069 T5907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2212076 T5938 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2212079 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2212079 T5907 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2212079 T5938 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2212079 T5938 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2089 name=collection1 org.apache.solr.core.SolrCore@4fd8f7e2 url=https://127.0.0.1:40243/md/tq/collection1 node=127.0.0.1:40243_md%2Ftq C2089_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:40243/md/tq, core=collection1, node_name=127.0.0.1:40243_md%2Ftq}
   [junit4]   2> 2212080 T5938 C2089 P40243 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:40243/md/tq/collection1/
   [junit4]   2> 2212080 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2212080 T5938 C2089 P40243 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2212081 T5938 C2089 P40243 oasc.SyncStrategy.syncToMe https://127.0.0.1:40243/md/tq/collection1/ has no replicas
   [junit4]   2> 2212081 T5938 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:40243/md/tq/collection1/ shard1
   [junit4]   2> 2212081 T5938 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 2212083 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2212084 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2212084 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2212089 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2212092 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2212194 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2212194 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2212242 T5938 oasc.ZkController.register We are https://127.0.0.1:40243/md/tq/collection1/ and leader is https://127.0.0.1:40243/md/tq/collection1/
   [junit4]   2> 2212242 T5938 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:40243/md/tq
   [junit4]   2> 2212242 T5938 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2212242 T5938 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 2212243 T5938 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2212244 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2212244 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2212244 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2212244 T5938 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2212246 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2212247 T5932 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:40243/md/tq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40243_md%2Ftq",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 2212257 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2212363 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2212363 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2212453 T5907 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 2212454 T5907 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2212459 T5907 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2212461 T5907 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:57283
   [junit4]   2> 2212464 T5907 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2212465 T5907 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2212465 T5907 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392783094181
   [junit4]   2> 2212466 T5907 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392783094181/'
   [junit4]   2> 2212506 T5907 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392783094181/solr.xml
   [junit4]   2> 2212606 T5907 oasc.CoreContainer.<init> New CoreContainer 22505925
   [junit4]   2> 2212607 T5907 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392783094181/]
   [junit4]   2> 2212609 T5907 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2212610 T5907 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2212610 T5907 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2212611 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2212611 T5907 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2212612 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2212612 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2212613 T5907 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2212613 T5907 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2212627 T5907 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2212627 T5907 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2212628 T5907 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2212628 T5907 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49190/solr
   [junit4]   2> 2212629 T5907 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2212630 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2212636 T5951 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e6b7e9d name:ZooKeeperConnection Watcher:127.0.0.1:49190 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2212637 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2212640 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2212642 T5953 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6697afdb name:ZooKeeperConnection Watcher:127.0.0.1:49190/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2212642 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2212653 T5907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2213657 T5907 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57283_md%2Ftq
   [junit4]   2> 2213660 T5907 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57283_md%2Ftq
   [junit4]   2> 2213663 T5940 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2213663 T5953 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2213663 T5931 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2213681 T5954 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2213681 T5954 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2213683 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2213683 T5954 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2213683 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2213683 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2213684 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2213685 T5932 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:57283/md/tq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57283_md%2Ftq",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2213685 T5932 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 2213685 T5932 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 2213694 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2213796 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2213796 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2213796 T5953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2214684 T5954 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2214684 T5954 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392783094181/collection1
   [junit4]   2> 2214684 T5954 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2214685 T5954 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2214686 T5954 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2214687 T5954 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2214687 T5954 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392783094181/collection1/'
   [junit4]   2> 2214689 T5954 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392783094181/collection1/lib/classes/' to classloader
   [junit4]   2> 2214690 T5954 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392783094181/collection1/lib/README' to classloader
   [junit4]   2> 2214759 T5954 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 2214832 T5954 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2214934 T5954 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2214950 T5954 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2215626 T5954 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2215627 T5954 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2215627 T5954 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2215640 T5954 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2215644 T5954 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2215658 T5954 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2215663 T5954 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2215668 T5954 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2215669 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2215669 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2215670 T5954 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2215671 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2215671 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2215671 T5954 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2215672 T5954 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392783094181/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty1/
   [junit4]   2> 2215672 T5954 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4533a8de
   [junit4]   2> 2215673 T5954 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty1
   [junit4]   2> 2215673 T5954 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty1/index/
   [junit4]   2> 2215674 T5954 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 2215674 T5954 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty1/index
   [junit4]   2> 2215675 T5954 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=59.2021484375, floorSegmentMB=1.990234375, forceMergeDeletesPctAllowed=18.048608453786404, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7408586669501135
   [junit4]   2> 2215676 T5954 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2ee1e582 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@77099598),segFN=segments_1,generation=1}
   [junit4]   2> 2215676 T5954 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2215681 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2215681 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2215682 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2215682 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2215682 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2215683 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2215683 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2215684 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2215684 T5954 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2215685 T5954 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2215686 T5954 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2215686 T5954 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2215686 T5954 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2215687 T5954 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2215688 T5954 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2215688 T5954 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2215705 T5954 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2215710 T5954 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2215710 T5954 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2215711 T5954 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=12.306640625, floorSegmentMB=0.96484375, forceMergeDeletesPctAllowed=1.5504112431909334, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.27580223564836803
   [junit4]   2> 2215712 T5954 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2ee1e582 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@77099598),segFN=segments_1,generation=1}
   [junit4]   2> 2215712 T5954 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2215713 T5954 oass.SolrIndexSearcher.<init> Opening Searcher@19291e79[collection1] main
   [junit4]   2> 2215718 T5955 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@19291e79[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2215721 T5954 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2215722 T5907 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 2215722 T5907 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2215722 T5958 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:57283/md/tq collection:collection1 shard:shard2
   [junit4]   2> 2215724 T5958 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 2215776 T5958 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 2215779 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215779 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215779 T5958 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2215780 T5958 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2215780 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C2090 name=collection1 org.apache.solr.core.SolrCore@23c4960d url=https://127.0.0.1:57283/md/tq/collection1 node=127.0.0.1:57283_md%2Ftq C2090_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:57283/md/tq, core=collection1, node_name=127.0.0.1:57283_md%2Ftq}
   [junit4]   2> 2215780 T5958 C2090 P57283 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:57283/md/tq/collection1/
   [junit4]   2> 2215781 T5958 C2090 P57283 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2215781 T5958 C2090 P57283 oasc.SyncStrategy.syncToMe https://127.0.0.1:57283/md/tq/collection1/ has no replicas
   [junit4]   2> 2215781 T5958 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:57283/md/tq/collection1/ shard2
   [junit4]   2> 2215781 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2215782 T5958 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 2215795 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215802 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2215802 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2215802 T5953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2215807 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2215816 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215923 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2215923 T5953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2215923 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2215962 T5958 oasc.ZkController.register We are https://127.0.0.1:57283/md/tq/collection1/ and leader is https://127.0.0.1:57283/md/tq/collection1/
   [junit4]   2> 2215962 T5958 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:57283/md/tq
   [junit4]   2> 2215962 T5958 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2215962 T5958 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 2215962 T5958 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2215964 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215964 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215964 T5958 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2215964 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215965 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2215966 T5932 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:57283/md/tq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:57283_md%2Ftq",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 2215975 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2216014 T5907 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 2216015 T5907 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2216020 T5907 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2216022 T5907 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:37421
   [junit4]   2> 2216025 T5907 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2216026 T5907 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2216026 T5907 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392783097827
   [junit4]   2> 2216027 T5907 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392783097827/'
   [junit4]   2> 2216067 T5907 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392783097827/solr.xml
   [junit4]   2> 2216079 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2216079 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2216079 T5953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2216165 T5907 oasc.CoreContainer.<init> New CoreContainer 1625424356
   [junit4]   2> 2216166 T5907 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392783097827/]
   [junit4]   2> 2216168 T5907 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2216168 T5907 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2216169 T5907 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2216169 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2216170 T5907 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2216170 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2216171 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2216171 T5907 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2216172 T5907 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2216187 T5907 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2216187 T5907 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2216188 T5907 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2216189 T5907 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49190/solr
   [junit4]   2> 2216189 T5907 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2216190 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2216193 T5969 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6260a433 name:ZooKeeperConnection Watcher:127.0.0.1:49190 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2216193 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2216196 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2216204 T5971 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2eabf257 name:ZooKeeperConnection Watcher:127.0.0.1:49190/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2216204 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2216209 T5907 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2217212 T5907 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37421_md%2Ftq
   [junit4]   2> 2217224 T5907 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37421_md%2Ftq
   [junit4]   2> 2217227 T5940 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2217228 T5953 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2217228 T5931 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2217227 T5971 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2217237 T5972 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2217237 T5972 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2217239 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2217239 T5972 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2217239 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2217239 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2217240 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2217241 T5932 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:37421/md/tq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37421_md%2Ftq",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2217241 T5932 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 2217241 T5932 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2217244 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2217349 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2217349 T5953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2217349 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2217349 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2218241 T5972 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2218241 T5972 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392783097827/collection1
   [junit4]   2> 2218241 T5972 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2218242 T5972 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2218242 T5972 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2218243 T5972 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2218244 T5972 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392783097827/collection1/'
   [junit4]   2> 2218245 T5972 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392783097827/collection1/lib/README' to classloader
   [junit4]   2> 2218246 T5972 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392783097827/collection1/lib/classes/' to classloader
   [junit4]   2> 2218315 T5972 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 2218388 T5972 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2218490 T5972 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2218506 T5972 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2219178 T5972 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2219179 T5972 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2219180 T5972 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2219193 T5972 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2219197 T5972 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2219210 T5972 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2219215 T5972 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2219220 T5972 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2219221 T5972 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2219221 T5972 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2219221 T5972 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2219222 T5972 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2219222 T5972 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2219223 T5972 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2219223 T5972 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392783097827/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty2/
   [junit4]   2> 2219224 T5972 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4533a8de
   [junit4]   2> 2219225 T5972 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty2
   [junit4]   2> 2219225 T5972 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty2/index/
   [junit4]   2> 2219226 T5972 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 2219226 T5972 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392783091015/jetty2/index
   [junit4]   2> 2219227 T5972 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=59.2021484375, floorSegmentMB=1.990234375, forceMergeDeletesPctAllowed=18.048608453786404, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7408586669501135
   [junit4]   2> 2219227 T5972 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3130e6d2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72d20376),segFN=segments_1,generation=1}
   [junit4]   2> 2219228 T5972 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2219232 T5972 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2219233 T5972 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2219233 T5972 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2219233 T5972 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2219234 T5972 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2219234 T5972 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2219235 T5972 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2219235 T5972 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2219235 T5972 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2219236 T5972 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2219237 T5972 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2219237 T5972 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2219238 T5972 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2219238 T5972 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2219239 T5972 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2219240 T5972 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2219255 T5972 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2219270 T5972 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2219270 T5972 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2219271 T5972 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=12.306640625, floorSegmentMB=0.96484375, forceMergeDeletesPctAllowed=1.5504112431909334, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.27580223564836803
   [junit4]   2> 2219272 T5972 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3130e6d2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@72d20376),segFN=segments_1,generation=1}
   [junit4]   2> 2219272 T5972 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2219272 T5972 oass.SolrIndexSearcher.<init> Opening Searcher@5bdd78aa[collection1] main
   [junit4]   2> 2219277 T5973 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5bdd78aa[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2219280 T5972 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2219281 T5907 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 2219282 T5907 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2219281 T5976 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:37421/md/tq collection:collection1 shard:shard1
   [junit4]   2> 2219283 T5976 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2219302 T5976 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2219304 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219305 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219305 T5976 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2219305 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219305 T5976 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2091 name=collection1 org.apache.solr.core.SolrCore@364cd2fd url=https://127.0.0.1:37421/md/tq/collection1 node=127.0.0.1:37421_md%2Ftq C2091_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:37421/md/tq, core=collection1, node_name=127.0.0.1:37421_md%2Ftq}
   [junit4]   2> 2219305 T5976 C2091 P37421 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:37421/md/tq/collection1/
   [junit4]   2> 2219306 T5976 C2091 P37421 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2219306 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2219306 T5976 C2091 P37421 oasc.SyncStrategy.syncToMe https://127.0.0.1:37421/md/tq/collection1/ has no replicas
   [junit4]   2> 2219307 T5976 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:37421/md/tq/collection1/ shard1
   [junit4]   2> 2219307 T5976 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2219310 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219311 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219311 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219311 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219311 T5953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219325 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2219328 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219432 T5953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219433 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219433 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219432 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219478 T5976 oasc.ZkController.register We are https://127.0.0.1:37421/md/tq/collection1/ and leader is https://127.0.0.1:37421/md/tq/collection1/
   [junit4]   2> 2219478 T5976 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:37421/md/tq
   [junit4]   2> 2219478 T5976 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2219478 T5976 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 2219479 T5976 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2219480 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219480 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219480 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219480 T5976 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2219482 T5932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2219482 T5932 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:37421/md/tq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37421_md%2Ftq",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 2219492 T5931 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219591 T5907 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 2219592 T5907 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2219597 T5907 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2219599 T5907 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:32879
   [junit4]   2> 2219602 T5907 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2219603 T5907 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2219603 T5907 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392783101386
   [junit4]   2> 2219604 T5907 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392783101386/'
   [junit4]   2> 2219621 T5953 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219621 T5931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219621 T5940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219621 T5971 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2219644 T5907 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392783101386/solr.xml
   [junit4]   2> 2219743 T5907 oasc.CoreContainer.<init> New CoreContainer 1949351145
   [junit4]   2> 2219744 T5907 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392783101386/]
   [junit4]   2> 2219746 T5907 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2219747 T5907 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2219747 T5907 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2219748 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2219748 T5907 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2219749 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2219749 T5907 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2219750 T5907 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2219750 T5907 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2219765 T5907 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2219766 T5907 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2219767 T5907 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2219767 T5907 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49190/solr
   [junit4]   2> 2219768 T5907 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2219769 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2219775 T5987 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@224a82ea name:ZooKeeperConnection Watcher:127.0.0.1:49190 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2219775 T5907 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2219779 T5907 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2219818 T5989 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@67ad3e71 name:ZooKeeperConnection Watcher:127.0.0.1:49190/solr got event

[...truncated too long message...]

A:
   [junit4]   1>        {"configName":"conf1"}
   [junit4]   1>     /solr/collections/control_collection/leader_elect (1)
   [junit4]   1>      /solr/collections/control_collection/leader_elect/shard1 (1)
   [junit4]   1>       /solr/collections/control_collection/leader_elect/shard1/election (1)
   [junit4]   1>        /solr/collections/control_collection/leader_elect/shard1/election/91277432653283332-core_node1-n_0000000000 (0)
   [junit4]   1>     /solr/collections/control_collection/leaders (1)
   [junit4]   1>      /solr/collections/control_collection/leaders/shard1 (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "node_name":"127.0.0.1:40243_md%2Ftq",
   [junit4]   1>            "core":"collection1",
   [junit4]   1>            "base_url":"https://127.0.0.1:40243/md/tq"}
   [junit4]   1>     /solr/collections/control_collection/shards (0)
   [junit4]   1>   /solr/live_nodes (5)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:27019_md%2Ftq (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:40243_md%2Ftq (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:37421_md%2Ftq (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:32879_md%2Ftq (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:57283_md%2Ftq (0)
   [junit4]   1>  /zookeeper (1)
   [junit4]   1>  DATA:
   [junit4]   1>      
   [junit4]   1> 
   [junit4]   1> found2: 4
   [junit4]   1> found3: 3
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BasicDistributedZkTest -Dtests.method=testDistribSearch -Dtests.seed=5CC03A27199D994 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es_DO -Dtests.timezone=America/Chicago -Dtests.file.encoding=UTF-8
   [junit4] ERROR    147s J0 | BasicDistributedZkTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:27019/md/tq/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5CC03A27199D994:842A8DBA06C6B9A8]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:503)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:168)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:146)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:663)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:724)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:150)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:121)
   [junit4]    > 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
   [junit4]    > 	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)
   [junit4]    > 	at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
   [junit4]    > 	... 47 more
   [junit4]   2> 2356004 T5907 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 147132 T5906 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> 2356920 T6009 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Appending, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=es_DO, timezone=America/Chicago
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=1,free=222491808,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [CoreContainerCoreInitFailuresTest, IndexBasedSpellCheckerTest, TestUniqueKeyFieldResource, BlockDirectoryTest, TestLazyCores, TestMaxScoreQueryParser, TestSolrXmlPersistor, OutputWriterTest, TriLevelCompositeIdRoutingTest, SuggesterWFSTTest, DateFieldTest, PathHierarchyTokenizerFactoryTest, FileUtilsTest, TestSystemIdResolver, TermsComponentTest, ShowFileRequestHandlerTest, TestSort, OverseerRolesTest, TestCollationKeyRangeQueries, TestNonDefinedSimilarityFactory, LeaderElectionIntegrationTest, TestDocSet, TestConfig, SolrCmdDistributorTest, TimeZoneUtilsTest, SpellCheckCollatorTest, StressHdfsTest, HdfsRecoveryZkTest, SystemInfoHandlerTest, TestArbitraryIndexDir, MinimalSchemaTest, UnloadDistributedZkTest, FullSolrCloudDistribCmdsTest, TestJmxIntegration, NoCacheHeaderTest, BlockCacheTest, UpdateRequestProcessorFactoryTest, RegexBoostProcessorTest, CacheHeaderTest, HighlighterTest, DirectSolrConnectionTest, TestSolrQueryParser, TestReload, TestPHPSerializedResponseWriter, NumericFieldsTest, QueryEqualityTest, SolrInfoMBeanTest, ChaosMonkeySafeLeaderTest, TestFiltering, StatsComponentTest, TestSchemaResource, TestFreeTextSuggestions, MBeansHandlerTest, TestRecovery, SignatureUpdateProcessorFactoryTest, PrimUtilsTest, CSVRequestHandlerTest, TestXIncludeConfig, TestFunctionQuery, ChangedSchemaMergeTest, SliceStateUpdateTest, TestStressVersions, DisMaxRequestHandlerTest, DOMUtilTest, SimpleFacetsTest, HighlighterConfigTest, DistributedQueryElevationComponentTest, ConvertedLegacyTest, TestBM25SimilarityFactory, ZkControllerTest, TestDocumentBuilder, TestExtendedDismaxParser, TestCloudManagedSchemaAddField, TestSolrIndexConfig, TestCSVResponseWriter, TestValueSourceCache, HdfsBasicDistributedZk2Test, TestPseudoReturnFields, BadCopyFieldTest, TestWriterPerf, TestRealTimeGet, TestCodecSupport, HdfsDirectoryTest, TestSchemaSimilarityResource, TestFastWriter, SpatialFilterTest, TestCoreContainer, DistribCursorPagingTest, CollectionsAPIDistributedZkTest, ChaosMonkeyNothingIsSafeTest, TestReplicationHandler, BasicDistributedZkTest]
   [junit4] Completed on J0 in 148.17s, 1 test, 1 error <<< FAILURES!

[...truncated 557 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1272: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:905: There were test failures: 362 suites, 1563 tests, 1 error, 35 ignored (7 assumptions)

Total time: 83 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1916 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
Thanks Mark!

On Sun, Mar 2, 2014 at 11:45 AM, Mark Miller <ma...@gmail.com> wrote:
> I just tried zapping the workspace for that job.
>
> - Mark
>
> http://about.me/markrmiller
>
> On Mar 2, 2014, at 11:44 AM, Mark Miller <ma...@gmail.com> wrote:
>
>> Yeah, I was just trying to figure this out too - best I can tell, that check out is in a bad state.
>>
>> - Mark
>>
>> http://about.me/markrmiller
>>
>> On Mar 2, 2014, at 11:14 AM, Robert Muir <rc...@gmail.com> wrote:
>>
>>> I think this one needs a clean-jars or similar on the workspace?
>>>
>>>
>>> On Sun, Mar 2, 2014 at 11:11 AM, Apache Jenkins Server
>>> <je...@builds.apache.org> wrote:
>>>> Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1916/
>>>>
>>>> All tests passed
>>>>
>>>> Build Log:
>>>> [...truncated 28344 lines...]
>>>> check-licenses:
>>>>    [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
>>>> [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
>>>> [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1
>>>>
>>>> [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
>>>> [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
>>>> [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
>>>> [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1
>>>>
>>>> [...truncated 3 lines...]
>>>> BUILD FAILED
>>>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
>>>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
>>>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
>>>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
>>>>
>>>> Total time: 111 minutes 2 seconds
>>>> Build step 'Invoke Ant' marked build as failure
>>>> Archiving artifacts
>>>> Recording test results
>>>> Email was triggered for: Failure
>>>> Sending email for trigger: Failure
>>>>
>>>>
>>>>
>>>>
>>>> ---------------------------------------------------------------------
>>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>>
>>> ---------------------------------------------------------------------
>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>>
>>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1916 - Still Failing

Posted by Mark Miller <ma...@gmail.com>.
I just tried zapping the workspace for that job.

- Mark

http://about.me/markrmiller

On Mar 2, 2014, at 11:44 AM, Mark Miller <ma...@gmail.com> wrote:

> Yeah, I was just trying to figure this out too - best I can tell, that check out is in a bad state.
> 
> - Mark
> 
> http://about.me/markrmiller
> 
> On Mar 2, 2014, at 11:14 AM, Robert Muir <rc...@gmail.com> wrote:
> 
>> I think this one needs a clean-jars or similar on the workspace?
>> 
>> 
>> On Sun, Mar 2, 2014 at 11:11 AM, Apache Jenkins Server
>> <je...@builds.apache.org> wrote:
>>> Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1916/
>>> 
>>> All tests passed
>>> 
>>> Build Log:
>>> [...truncated 28344 lines...]
>>> check-licenses:
>>>    [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
>>> [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
>>> [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1
>>> 
>>> [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
>>> [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
>>> [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
>>> [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1
>>> 
>>> [...truncated 3 lines...]
>>> BUILD FAILED
>>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
>>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
>>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
>>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
>>> 
>>> Total time: 111 minutes 2 seconds
>>> Build step 'Invoke Ant' marked build as failure
>>> Archiving artifacts
>>> Recording test results
>>> Email was triggered for: Failure
>>> Sending email for trigger: Failure
>>> 
>>> 
>>> 
>>> 
>>> ---------------------------------------------------------------------
>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>> For additional commands, e-mail: dev-help@lucene.apache.org
>> 
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>> 
> 


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1916 - Still Failing

Posted by Mark Miller <ma...@gmail.com>.
Yeah, I was just trying to figure this out too - best I can tell, that check out is in a bad state.

- Mark

http://about.me/markrmiller

On Mar 2, 2014, at 11:14 AM, Robert Muir <rc...@gmail.com> wrote:

> I think this one needs a clean-jars or similar on the workspace?
> 
> 
> On Sun, Mar 2, 2014 at 11:11 AM, Apache Jenkins Server
> <je...@builds.apache.org> wrote:
>> Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1916/
>> 
>> All tests passed
>> 
>> Build Log:
>> [...truncated 28344 lines...]
>> check-licenses:
>>     [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
>> [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
>> [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1
>> 
>> [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
>> [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
>> [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
>> [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1
>> 
>> [...truncated 3 lines...]
>> BUILD FAILED
>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
>> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
>> 
>> Total time: 111 minutes 2 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Archiving artifacts
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>> 
>> 
>> 
>> 
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
> 


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1916 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
I think this one needs a clean-jars or similar on the workspace?


On Sun, Mar 2, 2014 at 11:11 AM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1916/
>
> All tests passed
>
> Build Log:
> [...truncated 28344 lines...]
> check-licenses:
>      [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
>  [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
>  [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1
>
>  [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
>  [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
>  [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
>  [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1
>
> [...truncated 3 lines...]
> BUILD FAILED
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
>
> Total time: 111 minutes 2 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1916 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1916/

All tests passed

Build Log:
[...truncated 28344 lines...]
check-licenses:
     [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.

Total time: 111 minutes 2 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1915 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1915/

All tests passed

Build Log:
[...truncated 28367 lines...]
check-licenses:
     [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.

Total time: 103 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1914 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1914/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.OverseerTest.testOverseerFailure

Error Message:
KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1

Stack Trace:
org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
	at __randomizedtesting.SeedInfo.seed([9F02B25F17FA0C85:9B0A3DAC055FE3A4]:0)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
	at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:429)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:426)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:383)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:370)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:154)
	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:590)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)




Build Log:
[...truncated 10218 lines...]
   [junit4] Suite: org.apache.solr.cloud.OverseerTest
   [junit4]   2> 127058 T368 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 127059 T368 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-OverseerTest-1393611306913
   [junit4]   2> 127060 T368 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 127066 T368 oas.SolrTestCaseJ4.setUp ###Starting testOverseerFailure
   [junit4]   2> 127067 T368 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 127067 T369 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 127168 T368 oasc.ZkTestServer.run start zk server on port:14056
   [junit4]   2> 127169 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 127201 T375 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@129ace22 name:ZooKeeperConnection Watcher:127.0.0.1:14056/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127201 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 127202 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 127205 T377 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@365d1fe2 name:ZooKeeperConnection Watcher:127.0.0.1:14056 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127206 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 127215 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 127218 T379 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@267e5d35 name:ZooKeeperConnection Watcher:127.0.0.1:14056 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127219 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 127219 T368 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 127223 T368 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 127233 T368 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 127236 T368 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 127238 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 127242 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 127244 T381 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21065fec name:ZooKeeperConnection Watcher:127.0.0.1:14056/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127245 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 127246 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 127249 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 127252 T381 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 127252 T375 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 127253 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 127255 T383 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5a0a925d name:ZooKeeperConnection Watcher:127.0.0.1:14056/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 127255 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 127257 T368 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 127259 T368 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 127271 T368 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:14056_solr
   [junit4]   2> 127272 T368 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 127275 T368 oasc.Overseer.start Overseer (id=91331710610964484-127.0.0.1:14056_solr-n_0000000000) starting
   [junit4]   2> 127284 T368 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 127288 T368 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 127292 T368 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 127296 T385 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 127297 T385 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 127297 T384 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 128299 T383 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 128301 T384 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 128302 T384 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 128303 T384 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 128303 T384 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 128312 T383 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 128314 T381 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 128314 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 128800 T368 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 128828 T368 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 128835 T383 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 128836 T368 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 128837 T384 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 128840 T383 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 128840 T368 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 128842 T384 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 128843 T384 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 128846 T383 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 128949 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 128951 T381 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 130008 T368 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 130036 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130039 T387 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4befe1a1 name:ZooKeeperConnection Watcher:127.0.0.1:14056/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130039 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130051 T368 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:14056_solr
   [junit4]   2> 130051 T368 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 130054 T368 oasc.Overseer.start Overseer (id=91331710610964485-127.0.0.1:14056_solr-n_0000000001) starting
   [junit4]   2> 130071 T389 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 130072 T389 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 130072 T388 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 130075 T388 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 130076 T388 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 130076 T388 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 130080 T387 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 130081 T381 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 130081 T375 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 130137 T387 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 130137 T368 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 130139 T388 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 130152 T368 oasc.LeaderElector.checkIfIamLeader WARN Failed setting watch org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /collections/collection1/leader_elect/shard1/election/91331710610964483-node1_core1-n_0000000002
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1151)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:271)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:271)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:125)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:154)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:590)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:606)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 130161 T381 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 130163 T388 oasc.SolrException.log ERROR Problem cleaning up collection in zk:collection1:org.apache.zookeeper.KeeperException$NotEmptyException: KeeperErrorCode = Directory not empty for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:125)
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 		at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:174)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:171)
   [junit4]   2> 		at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:171)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:633)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:628)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:628)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:628)
   [junit4]   2> 		at org.apache.solr.cloud.Overseer$ClusterStateUpdater.removeCore(Overseer.java:949)
   [junit4]   2> 		at org.apache.solr.cloud.Overseer$ClusterStateUpdater.processMessage(Overseer.java:235)
   [junit4]   2> 		at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:192)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 	
   [junit4]   2> 130163 T368 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 130174 T387 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 130177 T387 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 130177 T375 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 130181 T368 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:14056 14056
   [junit4]   2> 130255 T368 oas.SolrTestCaseJ4.tearDown ###Ending testOverseerFailure
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=OverseerTest -Dtests.method=testOverseerFailure -Dtests.seed=9F02B25F17FA0C85 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=da -Dtests.timezone=EAT -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   3.26s J0 | OverseerTest.testOverseerFailure <<<
   [junit4]    > Throwable #1: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9F02B25F17FA0C85:9B0A3DAC055FE3A4]:0)
   [junit4]    > 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
   [junit4]    > 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]    > 	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:429)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:426)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:383)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:370)
   [junit4]    > 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
   [junit4]    > 	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:154)
   [junit4]    > 	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:590)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 130330 T368 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignmentBigger
   [junit4]   2> 130330 T368 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 130331 T390 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 130432 T368 oasc.ZkTestServer.run start zk server on port:41651
   [junit4]   2> 130433 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130439 T396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57fe2b6 name:ZooKeeperConnection Watcher:127.0.0.1:41651 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130440 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130493 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130501 T398 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b4985d2 name:ZooKeeperConnection Watcher:127.0.0.1:41651 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130501 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130502 T368 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 130506 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130515 T400 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c325a3b name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130515 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130516 T368 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 130519 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130521 T402 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45d7d580 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130521 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130523 T368 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 130526 T368 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 130531 T368 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:41651_solr
   [junit4]   2> 130531 T368 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 130534 T368 oasc.Overseer.start Overseer (id=91331710825070595-127.0.0.1:41651_solr-n_0000000000) starting
   [junit4]   2> 130537 T368 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 130541 T368 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 130545 T368 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 130548 T404 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 130549 T368 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 130549 T404 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 130549 T403 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 130553 T368 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 130563 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130566 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130569 T406 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b1451ed name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130569 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130571 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130574 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node0
   [junit4]   2> 130577 T400 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 130577 T406 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 130578 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130580 T408 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a71217a name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130580 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130582 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130585 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 130588 T408 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 130588 T406 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 130588 T400 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 130589 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130591 T410 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1209739c name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130591 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130593 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130596 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 130599 T406 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 130600 T408 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 130599 T410 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 130599 T400 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 130600 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130603 T412 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@183b5819 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130603 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130605 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130607 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node3
   [junit4]   2> 130611 T406 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 130611 T410 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 130611 T400 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 130611 T412 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 130611 T408 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 130612 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130614 T414 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3bd33b15 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130614 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130616 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130619 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node4
   [junit4]   2> 130622 T410 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 130622 T406 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 130622 T408 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 130622 T400 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 130622 T412 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 130622 T414 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 130623 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130625 T416 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c40a6f8 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130625 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130627 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130630 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node5
   [junit4]   2> 130633 T400 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 130633 T412 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 130633 T408 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 130633 T410 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 130633 T406 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 130634 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130634 T414 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 130634 T416 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 130637 T418 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1db9b30c name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130637 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130639 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130641 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node6
   [junit4]   2> 130645 T414 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 130645 T418 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 130645 T406 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 130646 T400 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 130645 T416 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 130645 T408 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 130645 T412 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 130645 T410 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 130646 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130648 T420 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61a05acb name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130649 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130650 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130653 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node7
   [junit4]   2> 130657 T400 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 130657 T410 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 130657 T416 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 130658 T408 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 130657 T414 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 130657 T418 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 130657 T406 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 130657 T412 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 130657 T420 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 130658 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130660 T422 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3bf7a927 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130661 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130662 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130665 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node8
   [junit4]   2> 130669 T408 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130669 T412 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130669 T400 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130670 T422 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130669 T410 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130669 T414 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130669 T406 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130669 T418 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130669 T416 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130670 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130670 T420 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 130673 T424 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ccefbbd name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130673 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130675 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130678 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node9
   [junit4]   2> 130681 T416 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130682 T412 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130682 T424 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130682 T422 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130682 T406 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130682 T414 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130681 T408 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130681 T418 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130683 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130682 T420 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130682 T400 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130682 T410 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 130685 T426 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6466fd34 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130686 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130688 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130690 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node10
   [junit4]   2> 130694 T418 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130694 T416 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130694 T424 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130695 T410 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130695 T412 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130694 T408 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130695 T400 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130695 T420 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130695 T422 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130694 T414 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130694 T426 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130694 T406 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 130696 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130698 T428 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3182afc9 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130699 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130700 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130703 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node11
   [junit4]   2> 130707 T422 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130707 T424 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130708 T410 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130707 T420 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130708 T412 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130707 T428 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130707 T414 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130707 T400 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130709 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130708 T418 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130708 T416 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130708 T406 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130708 T408 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130708 T426 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 130711 T430 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c2fe4e7 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130712 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130714 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130716 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node12
   [junit4]   2> 130720 T424 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130720 T430 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130721 T412 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130721 T400 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130721 T422 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130720 T414 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130721 T418 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130720 T408 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130720 T406 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130720 T416 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130722 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130721 T410 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130721 T426 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130721 T428 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130721 T420 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 130724 T432 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2fecb886 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130725 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130727 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130729 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node13
   [junit4]   2> 130733 T426 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130733 T412 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T424 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T418 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T428 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T410 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T432 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130733 T420 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130735 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130735 T430 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130735 T406 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T416 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T414 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T422 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T408 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130734 T400 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 130737 T434 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1672f0a6 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130739 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130740 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130743 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node14
   [junit4]   2> 130747 T406 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130747 T426 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130748 T420 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130748 T410 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130748 T432 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130748 T414 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130748 T408 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130747 T424 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130747 T434 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130749 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130749 T428 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130749 T400 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130749 T422 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130748 T412 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130748 T430 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130748 T418 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130748 T416 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 130751 T436 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@765746ca name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130753 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130754 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130757 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node15
   [junit4]   2> 130761 T414 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130761 T422 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130762 T430 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130762 T406 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130762 T420 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130762 T434 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130762 T416 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130761 T428 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130761 T400 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130761 T424 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130761 T408 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130761 T436 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130763 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130763 T432 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130762 T410 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130762 T412 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130762 T418 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130762 T426 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 130765 T438 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64862576 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130767 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130768 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130771 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node16
   [junit4]   2> 130775 T432 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T434 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T422 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T430 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T424 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130777 T438 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T418 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T420 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T410 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T412 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T406 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130776 T426 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130778 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130777 T408 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130780 T440 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12892419 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130777 T414 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130777 T428 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130777 T436 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130777 T416 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130777 T400 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 130781 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130784 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130787 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node17
   [junit4]   2> 130791 T406 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130791 T434 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130791 T418 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130792 T436 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130792 T420 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130792 T400 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130792 T424 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130793 T408 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130792 T428 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130792 T412 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130791 T410 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130791 T426 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130791 T432 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130791 T416 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130791 T440 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130795 T442 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68e6e00 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130793 T422 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130793 T430 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130792 T414 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130792 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130792 T438 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 130797 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130799 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130802 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node18
   [junit4]   2> 130806 T422 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130807 T436 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130807 T414 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130808 T420 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130808 T426 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130807 T438 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130807 T428 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130807 T430 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130807 T424 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130807 T400 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130809 T432 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130809 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130809 T440 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130808 T418 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130808 T406 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130808 T434 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130808 T416 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130808 T412 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130808 T410 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130808 T442 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130807 T408 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 130811 T444 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@97d9475 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130813 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130815 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130818 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node19
   [junit4]   2> 130822 T420 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130823 T426 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130823 T440 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130823 T434 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130824 T418 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130824 T438 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130824 T430 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130823 T406 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130823 T412 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130823 T442 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130823 T410 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130823 T432 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130823 T416 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130825 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130825 T444 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130825 T436 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130824 T408 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130824 T414 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130824 T424 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130824 T400 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130824 T422 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130824 T428 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 130828 T446 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3076f54c name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130831 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130834 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130837 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node20
   [junit4]   2> 130842 T428 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T444 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T432 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T412 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T430 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T438 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130844 T406 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130842 T424 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130842 T400 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130845 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130844 T446 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130844 T434 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130844 T410 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130844 T416 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130844 T420 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130844 T414 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130844 T418 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130844 T442 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T440 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T436 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T408 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T422 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130843 T426 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 130848 T448 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a40ce0a name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130850 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130852 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130855 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node21
   [junit4]   2> 130859 T446 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130860 T410 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130860 T420 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130860 T416 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130860 T440 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130860 T418 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130859 T406 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130859 T448 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130859 T434 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130862 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130862 T428 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T430 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T400 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T424 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T422 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T436 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T408 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T438 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T444 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T414 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130861 T432 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130860 T412 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130860 T442 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130860 T426 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 130865 T450 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c4b88a5 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130868 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130869 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130872 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node22
   [junit4]   2> 130877 T406 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130877 T416 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T408 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T422 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T432 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130879 T444 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130879 T420 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130877 T446 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130879 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130877 T426 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130877 T448 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130877 T450 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130877 T434 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130877 T418 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130879 T410 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130882 T452 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16d11a0e name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130879 T412 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130879 T430 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T440 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T436 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T414 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T442 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T400 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T428 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T424 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130878 T438 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 130882 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130887 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130890 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node23
   [junit4]   2> 130909 T408 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130909 T424 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130910 T452 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130910 T428 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130911 T400 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130911 T412 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130911 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130910 T430 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130910 T450 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130910 T436 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130910 T414 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130910 T442 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130910 T406 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130914 T454 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e709107 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130910 T434 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130910 T418 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130910 T422 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130909 T416 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130909 T426 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130909 T444 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130909 T432 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130909 T438 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130914 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130911 T446 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130911 T448 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130911 T420 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130911 T410 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130911 T440 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 130918 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130921 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node24
   [junit4]   2> 130926 T408 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130926 T426 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130926 T406 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T434 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T420 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T440 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T452 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T400 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T430 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T428 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T422 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130926 T418 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130926 T416 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130926 T432 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130926 T438 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130926 T444 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130926 T424 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130929 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130928 T410 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130928 T414 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130928 T454 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130928 T442 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130928 T446 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130928 T448 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130928 T412 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T436 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130927 T450 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 130931 T456 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72e0e90a name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130935 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130936 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130939 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node25
   [junit4]   2> 130943 T410 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T412 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130943 T420 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T422 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T424 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T434 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T414 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130946 T452 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T432 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T440 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T444 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T408 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T406 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T430 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T428 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T400 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T456 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T442 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T446 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T448 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130944 T450 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130949 T458 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5cd2aec1 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130947 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130946 T418 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130946 T426 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130946 T436 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130946 T416 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T454 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130945 T438 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 130951 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130954 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130957 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node26
   [junit4]   2> 130961 T408 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T424 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T458 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T420 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130961 T426 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130963 T428 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130961 T444 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130964 T452 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130964 T430 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130964 T422 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130963 T450 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130963 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130963 T446 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130963 T416 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130963 T432 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130963 T438 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130963 T448 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130963 T410 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T440 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T456 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T412 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T442 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T414 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T406 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T400 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T418 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130962 T434 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130966 T460 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e09645c name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130964 T454 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130964 T436 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 130970 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130972 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130975 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node27
   [junit4]   2> 130980 T416 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130980 T438 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130980 T424 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T442 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T444 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130980 T418 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130982 T440 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130982 T436 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130982 T448 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130983 T400 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130980 T406 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130983 T368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130980 T432 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130982 T414 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130982 T422 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130982 T454 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130982 T452 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130982 T446 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130986 T462 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@42ff95b0 name:ZooKeeperConnection Watcher:127.0.0.1:41651/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130982 T458 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130982 T430 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T420 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T412 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T428 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T410 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T450 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T456 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T460 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130981 T408 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130980 T434 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130980 T426 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 130986 T368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130991 T368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 130994 T368 oascc.SolrZkClient.makePath makePath: /live_nodes/node28
   [junit4]   2> 130998 T460 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 130998 T440 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 130998 T412 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 130998 T448 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 131000 T430 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 131000 T422 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 131000 T434 oascc.ZkState

[...truncated too long message...]

nit4]   2> 152204 T368 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 152208 T368 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:26420_solr
   [junit4]   2> 152208 T368 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 152210 T368 oasc.Overseer.start Overseer (id=91331712246415364-127.0.0.1:26420_solr-n_0000000000) starting
   [junit4]   2> 152212 T368 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 152215 T368 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 152218 T368 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 152220 T898 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 152220 T898 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 152220 T897 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 152222 T897 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 152223 T897 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=12 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"12",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 152223 T897 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8, shard9, shard10, shard11, shard12]
   [junit4]   2> 152224 T897 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard10
   [junit4]   2> 152226 T896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 152227 T894 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 152227 T888 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 152723 T368 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard10/election
   [junit4]   2> 152753 T368 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard10
   [junit4]   2> 152766 T896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 152766 T368 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 152767 T897 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 152775 T888 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 152775 T894 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 152778 T368 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26420 26420
   [junit4]   2> 152843 T368 oas.SolrTestCaseJ4.tearDown ###Ending testPlaceholders
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 152846 T368 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 152846 T368 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 153322 T881 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 153322 T881 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 153323 T881 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 155847 T368 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene46: {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=da, timezone=EAT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=1,free=75672752,total=145752064
   [junit4]   2> NOTE: All tests run in this JVM: [AlternateDirectoryTest, TestRemoteStreaming, TestSystemIdResolver, TestCloudManagedSchemaAddField, TestJmxMonitoredMap, TestValueSourceCache, CircularListTest, ClusterStateUpdateTest, TestCollationField, QueryElevationComponentTest, SuggestComponentTest, RequestHandlersTest, TestSchemaResource, OverseerTest]
   [junit4] Completed on J0 in 28.84s, 8 tests, 1 error <<< FAILURES!

[...truncated 987 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:451: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:490: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1278: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:911: There were test failures: 363 suites, 1570 tests, 1 error, 36 ignored (8 assumptions)

Total time: 63 minutes 58 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1913 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1913/

All tests passed

Build Log:
[...truncated 28417 lines...]
check-licenses:
     [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.

Total time: 108 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1912 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
I committed a fix.

On Wed, Feb 26, 2014 at 6:01 PM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1912/
>
> All tests passed
>
> Build Log:
> [...truncated 28267 lines...]
> check-licenses:
>      [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
>  [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
>  [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1
>
>  [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
>  [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
>  [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
>  [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1
>
> [...truncated 3 lines...]
> BUILD FAILED
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
> /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.
>
> Total time: 107 minutes 7 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1912 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1912/

All tests passed

Build Log:
[...truncated 28267 lines...]
check-licenses:
     [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.

Total time: 107 minutes 7 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1911 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1911/

3 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([4D75887B35EB759B]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([4D75887B35EB759B]:0)


REGRESSION:  org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest.testDistribSearch

Error Message:
this codec cannot write docvalues

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: this codec cannot write docvalues
	at __randomizedtesting.SeedInfo.seed([4D75887B35EB759B:CC93066342B415A7]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:497)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:168)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:146)
	at org.apache.solr.BaseDistributedSearchTestCase.commit(BaseDistributedSearchTestCase.java:505)
	at org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest.doTest(DistributedQueryComponentOptimizationTest.java:66)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)




Build Log:
[...truncated 10577 lines...]
   [junit4] Suite: org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest
   [junit4]   2> 4988487 T16025 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 4988488 T16025 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /v_fzy/e
   [junit4]   2> 4988491 T16025 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-DistributedQueryComponentOptimizationTest-1393371199395
   [junit4]   2> 4988492 T16025 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 4988493 T16025 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 4988494 T16025 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 4988495 T16025 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 4988643 T16025 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 4988741 T16025 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4988743 T16025 oass.IndexSchema.readSchema Reading Solr Schema from schema-custom-field.xml
   [junit4]   2> 4988750 T16025 oass.IndexSchema.readSchema [null] Schema name=test-custom-field-sort
   [junit4]   2> 4988793 T16025 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 4988797 T16025 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4988799 T16025 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 4988799 T16025 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 4988800 T16025 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 4988894 T16025 oasc.CoreContainer.<init> New CoreContainer 1514486332
   [junit4]   2> 4988895 T16025 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 4988897 T16025 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 4988897 T16025 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 4988898 T16025 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 4988898 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4988899 T16025 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4988899 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4988900 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4988900 T16025 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4988901 T16025 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4988917 T16025 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4988917 T16025 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4988917 T16025 oasc.CoreContainer.load Host Name: 
   [junit4]   2> 4988924 T16026 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 4988925 T16026 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 4988926 T16026 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 4988927 T16026 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 4988927 T16026 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 4989010 T16026 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 4989107 T16026 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4989108 T16026 oass.IndexSchema.readSchema Reading Solr Schema from schema-custom-field.xml
   [junit4]   2> 4989115 T16026 oass.IndexSchema.readSchema [collection1] Schema name=test-custom-field-sort
   [junit4]   2> 4989155 T16026 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 4989158 T16026 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4989160 T16026 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4989161 T16026 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-DistributedQueryComponentOptimizationTest-1393371199395/
   [junit4]   2> 4989161 T16026 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5738ff7c
   [junit4]   2> 4989162 T16026 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-DistributedQueryComponentOptimizationTest-1393371199395
   [junit4]   2> 4989163 T16026 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-DistributedQueryComponentOptimizationTest-1393371199395/index/
   [junit4]   2> 4989163 T16026 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-DistributedQueryComponentOptimizationTest-1393371199395/index' doesn't exist. Creating new index...
   [junit4]   2> 4989164 T16026 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-DistributedQueryComponentOptimizationTest-1393371199395/index
   [junit4]   2> 4989164 T16026 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24571304246398445]
   [junit4]   2> 4989165 T16026 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@37fa5b99 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2d6d88ea)),segFN=segments_1,generation=1}
   [junit4]   2> 4989166 T16026 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4989167 T16026 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 4989171 T16026 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 4989172 T16026 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 4989172 T16026 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4989173 T16026 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4989173 T16026 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4989173 T16026 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4989174 T16026 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4989174 T16026 oasup.UniqFieldsUpdateProcessorFactory.init WARN Use of the 'fields' init param in UniqFieldsUpdateProcessorFactory is deprecated, please use 'fieldName' (or another FieldMutatingUpdateProcessorFactory selector option) instead
   [junit4]   2> 4989175 T16026 oasup.UniqFieldsUpdateProcessorFactory.init Replacing 'fields' init param with (individual) 'fieldName' params
   [junit4]   2> 4989175 T16026 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4989176 T16026 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 4989176 T16026 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4989177 T16026 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4989177 T16026 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4989178 T16026 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 4989178 T16026 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 4989179 T16026 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 4989180 T16026 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 4989180 T16026 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 4989181 T16026 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 4989182 T16026 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 4989182 T16026 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 4989183 T16026 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 4989183 T16026 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 4989184 T16026 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 4989184 T16026 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 4989185 T16026 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989186 T16026 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989186 T16026 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989187 T16026 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989188 T16026 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989188 T16026 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989189 T16026 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989189 T16026 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989190 T16026 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 4989191 T16026 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 4989208 T16026 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4989210 T16026 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 4989214 T16026 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4989214 T16026 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4989215 T16026 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=55.7451171875, floorSegmentMB=0.6640625, forceMergeDeletesPctAllowed=29.335149011310897, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 4989216 T16026 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@37fa5b99 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2d6d88ea)),segFN=segments_1,generation=1}
   [junit4]   2> 4989216 T16026 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4989217 T16026 oass.SolrIndexSearcher.<init> Opening Searcher@32b06c9b[collection1] main
   [junit4]   2> 4989217 T16026 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 4989225 T16026 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 4989306 T16027 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@32b06c9b[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 4989313 T16026 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4989330 T16025 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 4989338 T16025 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 4989342 T16025 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 4989347 T16025 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 4989351 T16025 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:63664
   [junit4]   2> 4989354 T16025 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 4989354 T16025 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 4989355 T16025 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 4989356 T16025 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 4989404 T16025 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/solr.xml
   [junit4]   2> 4989562 T16025 oasc.CoreContainer.<init> New CoreContainer 1766085693
   [junit4]   2> 4989564 T16025 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 4989566 T16025 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4989566 T16025 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: https
   [junit4]   2> 4989567 T16025 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4989568 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4989568 T16025 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4989569 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4989569 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4989570 T16025 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4989571 T16025 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4989586 T16025 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4989587 T16025 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4989588 T16025 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 4989594 T16038 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 4989595 T16038 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 4989596 T16038 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 4989597 T16038 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 4989598 T16038 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 4989688 T16038 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 4989785 T16038 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4989786 T16038 oass.IndexSchema.readSchema Reading Solr Schema from schema-custom-field.xml
   [junit4]   2> 4989794 T16038 oass.IndexSchema.readSchema [collection1] Schema name=test-custom-field-sort
   [junit4]   2> 4989836 T16038 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 4989840 T16038 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4989841 T16038 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4989842 T16038 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/control/data/
   [junit4]   2> 4989842 T16038 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5738ff7c
   [junit4]   2> 4989844 T16038 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/control/data
   [junit4]   2> 4989844 T16038 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/control/data/index/
   [junit4]   2> 4989844 T16038 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4989845 T16038 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/control/data/index
   [junit4]   2> 4989846 T16038 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24571304246398445]
   [junit4]   2> 4989847 T16038 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@1f10229f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2311d9c2)),segFN=segments_1,generation=1}
   [junit4]   2> 4989847 T16038 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4989848 T16038 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 4989853 T16038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 4989854 T16038 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 4989854 T16038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4989855 T16038 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4989855 T16038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4989856 T16038 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4989856 T16038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4989857 T16038 oasup.UniqFieldsUpdateProcessorFactory.init WARN Use of the 'fields' init param in UniqFieldsUpdateProcessorFactory is deprecated, please use 'fieldName' (or another FieldMutatingUpdateProcessorFactory selector option) instead
   [junit4]   2> 4989857 T16038 oasup.UniqFieldsUpdateProcessorFactory.init Replacing 'fields' init param with (individual) 'fieldName' params
   [junit4]   2> 4989857 T16038 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4989858 T16038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 4989859 T16038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4989859 T16038 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4989859 T16038 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4989860 T16038 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 4989860 T16038 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 4989861 T16038 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 4989862 T16038 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 4989863 T16038 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 4989864 T16038 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 4989865 T16038 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 4989866 T16038 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 4989867 T16038 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 4989867 T16038 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 4989867 T16038 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 4989868 T16038 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 4989869 T16038 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989870 T16038 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989871 T16038 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989871 T16038 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989872 T16038 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989873 T16038 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989874 T16038 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989875 T16038 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4989875 T16038 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 4989876 T16038 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 4989879 T16038 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4989881 T16038 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 4989887 T16038 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4989887 T16038 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4989888 T16038 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=55.7451171875, floorSegmentMB=0.6640625, forceMergeDeletesPctAllowed=29.335149011310897, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 4989889 T16038 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@1f10229f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2311d9c2)),segFN=segments_1,generation=1}
   [junit4]   2> 4989890 T16038 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4989890 T16038 oass.SolrIndexSearcher.<init> Opening Searcher@3f2be016[collection1] main
   [junit4]   2> 4989891 T16038 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 4989926 T16038 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 4989987 T16039 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3f2be016[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 4989996 T16038 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4989996 T16025 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 4989997 T16025 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 4990002 T16025 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 4990007 T16025 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 4990010 T16025 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:47131
   [junit4]   2> 4990014 T16025 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 4990014 T16025 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 4990015 T16025 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 4990015 T16025 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 4990064 T16025 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/solr.xml
   [junit4]   2> 4990163 T16025 oasc.CoreContainer.<init> New CoreContainer 1449787431
   [junit4]   2> 4990165 T16025 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 4990167 T16025 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4990167 T16025 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: https
   [junit4]   2> 4990168 T16025 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4990169 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4990169 T16025 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4990170 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4990171 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4990171 T16025 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4990172 T16025 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4990188 T16025 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4990189 T16025 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4990189 T16025 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 4990196 T16050 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 4990196 T16050 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 4990198 T16050 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 4990199 T16050 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 4990199 T16050 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 4990289 T16050 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 4990386 T16050 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4990387 T16050 oass.IndexSchema.readSchema Reading Solr Schema from schema-custom-field.xml
   [junit4]   2> 4990394 T16050 oass.IndexSchema.readSchema [collection1] Schema name=test-custom-field-sort
   [junit4]   2> 4990436 T16050 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 4990439 T16050 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4990441 T16050 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4990442 T16050 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard0/data/
   [junit4]   2> 4990442 T16050 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5738ff7c
   [junit4]   2> 4990443 T16050 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard0/data
   [junit4]   2> 4990444 T16050 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard0/data/index/
   [junit4]   2> 4990444 T16050 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard0/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4990445 T16050 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard0/data/index
   [junit4]   2> 4990445 T16050 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24571304246398445]
   [junit4]   2> 4990447 T16050 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@3ab4eb86 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1abf9b2a)),segFN=segments_1,generation=1}
   [junit4]   2> 4990447 T16050 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4990448 T16050 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 4990453 T16050 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 4990454 T16050 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 4990454 T16050 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4990455 T16050 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4990455 T16050 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4990455 T16050 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4990456 T16050 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4990456 T16050 oasup.UniqFieldsUpdateProcessorFactory.init WARN Use of the 'fields' init param in UniqFieldsUpdateProcessorFactory is deprecated, please use 'fieldName' (or another FieldMutatingUpdateProcessorFactory selector option) instead
   [junit4]   2> 4990457 T16050 oasup.UniqFieldsUpdateProcessorFactory.init Replacing 'fields' init param with (individual) 'fieldName' params
   [junit4]   2> 4990457 T16050 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4990457 T16050 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 4990458 T16050 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4990459 T16050 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4990459 T16050 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4990459 T16050 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 4990460 T16050 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 4990461 T16050 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 4990462 T16050 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 4990463 T16050 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 4990464 T16050 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 4990464 T16050 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 4990465 T16050 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 4990466 T16050 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 4990466 T16050 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 4990467 T16050 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 4990468 T16050 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 4990468 T16050 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4990469 T16050 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4990470 T16050 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4990471 T16050 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4990472 T16050 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4990472 T16050 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4990473 T16050 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4990474 T16050 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4990475 T16050 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 4990476 T16050 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 4990493 T16050 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4990495 T16050 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 4990501 T16050 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4990501 T16050 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4990502 T16050 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=55.7451171875, floorSegmentMB=0.6640625, forceMergeDeletesPctAllowed=29.335149011310897, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 4990503 T16050 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@3ab4eb86 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1abf9b2a)),segFN=segments_1,generation=1}
   [junit4]   2> 4990503 T16050 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4990504 T16050 oass.SolrIndexSearcher.<init> Opening Searcher@51f0468a[collection1] main
   [junit4]   2> 4990504 T16050 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 4990541 T16050 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 4990601 T16051 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@51f0468a[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 4990610 T16050 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4990610 T16025 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 4990611 T16025 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 4990616 T16025 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 4990621 T16025 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 4990624 T16025 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:62277
   [junit4]   2> 4990627 T16025 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 4990628 T16025 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 4990628 T16025 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 4990629 T16025 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 4990677 T16025 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/solr.xml
   [junit4]   2> 4990777 T16025 oasc.CoreContainer.<init> New CoreContainer 1296771769
   [junit4]   2> 4990778 T16025 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 4990780 T16025 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4990781 T16025 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: https
   [junit4]   2> 4990782 T16025 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4990782 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4990783 T16025 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4990783 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4990784 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4990785 T16025 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4990785 T16025 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4990801 T16025 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4990802 T16025 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4990802 T16025 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 4990809 T16062 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 4990809 T16062 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 4990811 T16062 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 4990811 T16062 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 4990812 T16062 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 4990902 T16062 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 4991046 T16062 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4991048 T16062 oass.IndexSchema.readSchema Reading Solr Schema from schema-custom-field.xml
   [junit4]   2> 4991055 T16062 oass.IndexSchema.readSchema [collection1] Schema name=test-custom-field-sort
   [junit4]   2> 4991097 T16062 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 4991100 T16062 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4991102 T16062 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4991102 T16062 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard1/data/
   [junit4]   2> 4991103 T16062 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5738ff7c
   [junit4]   2> 4991104 T16062 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard1/data
   [junit4]   2> 4991104 T16062 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard1/data/index/
   [junit4]   2> 4991105 T16062 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4991105 T16062 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard1/data/index
   [junit4]   2> 4991106 T16062 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24571304246398445]
   [junit4]   2> 4991107 T16062 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@50a3e688 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f86d67)),segFN=segments_1,generation=1}
   [junit4]   2> 4991108 T16062 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4991108 T16062 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 4991114 T16062 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 4991114 T16062 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 4991115 T16062 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4991115 T16062 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4991116 T16062 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4991116 T16062 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4991116 T16062 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4991117 T16062 oasup.UniqFieldsUpdateProcessorFactory.init WARN Use of the 'fields' init param in UniqFieldsUpdateProcessorFactory is deprecated, please use 'fieldName' (or another FieldMutatingUpdateProcessorFactory selector option) instead
   [junit4]   2> 4991117 T16062 oasup.UniqFieldsUpdateProcessorFactory.init Replacing 'fields' init param with (individual) 'fieldName' params
   [junit4]   2> 4991118 T16062 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4991118 T16062 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 4991119 T16062 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4991119 T16062 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4991120 T16062 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4991120 T16062 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 4991121 T16062 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 4991122 T16062 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 4991122 T16062 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 4991123 T16062 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 4991124 T16062 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 4991125 T16062 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 4991126 T16062 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 4991127 T16062 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 4991127 T16062 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 4991127 T16062 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 4991128 T16062 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 4991129 T16062 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991130 T16062 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991131 T16062 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991131 T16062 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991132 T16062 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991133 T16062 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991134 T16062 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991134 T16062 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991135 T16062 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 4991136 T16062 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 4991153 T16062 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4991155 T16062 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 4991161 T16062 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4991161 T16062 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4991162 T16062 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=55.7451171875, floorSegmentMB=0.6640625, forceMergeDeletesPctAllowed=29.335149011310897, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 4991163 T16062 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@50a3e688 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1f86d67)),segFN=segments_1,generation=1}
   [junit4]   2> 4991164 T16062 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4991164 T16062 oass.SolrIndexSearcher.<init> Opening Searcher@58f4316a[collection1] main
   [junit4]   2> 4991165 T16062 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 4991198 T16062 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 4991259 T16063 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@58f4316a[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 4991267 T16062 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4991268 T16025 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 4991269 T16025 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 4991274 T16025 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 4991278 T16025 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 4991282 T16025 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:54201
   [junit4]   2> 4991285 T16025 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 4991285 T16025 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 4991286 T16025 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr
   [junit4]   2> 4991287 T16025 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/'
   [junit4]   2> 4991336 T16025 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/solr.xml
   [junit4]   2> 4991435 T16025 oasc.CoreContainer.<init> New CoreContainer 352063868
   [junit4]   2> 4991437 T16025 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/]
   [junit4]   2> 4991439 T16025 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 4991439 T16025 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: https
   [junit4]   2> 4991440 T16025 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 4991441 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 4991441 T16025 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 4991442 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 4991442 T16025 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 4991443 T16025 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 4991444 T16025 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 4991458 T16025 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 4991458 T16025 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 4991459 T16025 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 4991465 T16074 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1
   [junit4]   2> 4991466 T16074 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 4991467 T16074 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 4991468 T16074 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 4991469 T16074 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 4991558 T16074 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 4991655 T16074 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 4991656 T16074 oass.IndexSchema.readSchema Reading Solr Schema from schema-custom-field.xml
   [junit4]   2> 4991663 T16074 oass.IndexSchema.readSchema [collection1] Schema name=test-custom-field-sort
   [junit4]   2> 4991705 T16074 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 4991709 T16074 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 4991710 T16074 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 4991711 T16074 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/, dataDir=./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard2/data/
   [junit4]   2> 4991712 T16074 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5738ff7c
   [junit4]   2> 4991713 T16074 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard2/data
   [junit4]   2> 4991713 T16074 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard2/data/index/
   [junit4]   2> 4991714 T16074 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 4991714 T16074 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.handler.component.DistributedQueryComponentOptimizationTest-1393371200243/shard2/data/index
   [junit4]   2> 4991715 T16074 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=27, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24571304246398445]
   [junit4]   2> 4991716 T16074 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@39d641b3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3d6aa46d)),segFN=segments_1,generation=1}
   [junit4]   2> 4991717 T16074 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4991717 T16074 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 4991722 T16074 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 4991723 T16074 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 4991724 T16074 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4991724 T16074 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 4991724 T16074 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4991725 T16074 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 4991725 T16074 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4991726 T16074 oasup.UniqFieldsUpdateProcessorFactory.init WARN Use of the 'fields' init param in UniqFieldsUpdateProcessorFactory is deprecated, please use 'fieldName' (or another FieldMutatingUpdateProcessorFactory selector option) instead
   [junit4]   2> 4991726 T16074 oasup.UniqFieldsUpdateProcessorFactory.init Replacing 'fields' init param with (individual) 'fieldName' params
   [junit4]   2> 4991727 T16074 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 4991727 T16074 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 4991728 T16074 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4991728 T16074 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 4991729 T16074 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 4991729 T16074 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 4991729 T16074 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 4991730 T16074 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 4991731 T16074 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 4991732 T16074 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 4991733 T16074 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 4991734 T16074 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 4991735 T16074 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 4991735 T16074 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 4991736 T16074 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 4991736 T16074 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 4991737 T16074 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 4991738 T16074 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991739 T16074 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991739 T16074 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991740 T16074 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991741 T16074 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991742 T16074 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991742 T16074 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991743 T16074 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 4991744 T16074 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 4991745 T16074 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 4991762 T16074 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 4991764 T16074 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 4991770 T16074 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 4991770 T16074 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 4991771 T16074 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=27, maxMergedSegmentMB=55.7451171875, floorSegmentMB=0.6640625, forceMergeDeletesPctAllowed=29.335149011310897, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 4991772 T16074 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(RAMDirectory@39d641b3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3d6aa46d)),segFN=segments_1,generation=1}
   [junit4]   2> 4991773 T16074 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 4991773 T16074 oass.SolrIndexSearcher.<init> Opening Searcher@43771604[collection1] main
   [junit4]   2> 4991774 T16074 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 4991782 T16074 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 4991842 T16075 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@43771604[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 4991851 T16074 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 4991852 T16025 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 4991852 T16025 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C2695 name=collection1 org.apache.solr.core.SolrCore@7517a0df
   [junit4]   2> 4991987 T16031 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1461055602441584640)} 0 1
   [junit4]   2> ASYNC  NEW_CORE C2696 name=collection1 org.apache.solr.core.SolrCore@47d5a92f
   [junit4]   2> 4992117 T16043 C2696 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1461055602577899520)} 0 1
   [junit4]   2> ASYNC  NEW_CORE C2697 name=collection1 org.apache.solr.core.SolrCore@70a8d3e8
   [junit4]   2> 4992248 T16055 C2697 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1461055602715262976)} 0 1
   [junit4]   2> ASYNC  NEW_CORE C2698 name=collection1 org.apache.solr.core.SolrCore@71545e64
   [junit4]   2> 4992379 T16067 C2698 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1461055602852626432)} 0 1
   [junit4]   2> 4992385 T16032 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[1 (1461055602858917888)]} 0 1
   [junit4]   2> 4992389 T16044 C2696 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[1 (1461055602863112192)]} 0 1
   [junit4]   2> 4992393 T16033 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[2 (1461055602868355072)]} 0 0
   [junit4]   2> 4992397 T16056 C2697 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[2 (1461055602871500800)]} 0 1
   [junit4]   2> 4992400 T16034 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[3 (1461055602875695104)]} 0 0
   [junit4]   2> 4992405 T16068 C2698 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[3 (1461055602879889408)]} 0 1
   [junit4]   2> 4992408 T16035 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[4 (1461055602884083712)]} 0 0
   [junit4]   2> 4992412 T16045 C2696 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[4 (1461055602888278016)]} 0 0
   [junit4]   2> 4992416 T16036 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[5 (1461055602891423744)]} 0 1
   [junit4]   2> 4992419 T16057 C2697 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[5 (1461055602895618048)]} 0 0
   [junit4]   2> 4992423 T16037 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[6 (1461055602899812352)]} 0 0
   [junit4]   2> 4992427 T16069 C2698 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[6 (1461055602904006656)]} 0 0
   [junit4]   2> 4992431 T16093 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[7 (1461055602908200960)]} 0 0
   [junit4]   2> 4992435 T16046 C2696 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[7 (1461055602912395264)]} 0 0
   [junit4]   2> 4992439 T16096 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[8 (1461055602916589568)]} 0 0
   [junit4]   2> 4992443 T16058 C2697 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[8 (1461055602919735296)]} 0 1
   [junit4]   2> 4992446 T16099 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[9 (1461055602923929600)]} 0 0
   [junit4]   2> 4992450 T16070 C2698 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[9 (1461055602928123904)]} 0 0
   [junit4]   2> 4992454 T16102 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[10 (1461055602932318208)]} 0 0
   [junit4]   2> 4992458 T16047 C2696 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[10 (1461055602936512512)]} 0 0
   [junit4]   2> 4992462 T16105 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[11 (1461055602940706816)]} 0 0
   [junit4]   2> 4992466 T16059 C2697 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[11 (1461055602944901120)]} 0 1
   [junit4]   2> 4992470 T16108 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[12 (1461055602948046848)]} 0 1
   [junit4]   2> 4992473 T16071 C2698 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[12 (1461055602952241152)]} 0 0
   [junit4]   2> 4992477 T16111 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[13 (1461055602956435456)]} 0 0
   [junit4]   2> 4992481 T16048 C2696 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&version=2} {add=[13 (1461055602960629760)]} 0 0
   [junit4]   2> 4992485 T16114 C2695 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 4992486 T16114 C2695 oasup.LogUpdateProcessor.finish [collection1] webapp=/v_fzy/e path=/update params={wt=javabin&waitSearcher=true&softCommit=false&commit=true&version=2} {} 0 1
   [junit4]   2> 4992487 T16114 C2695 oasc.SolrException.log ERROR java.lang.UnsupportedOperationException: this codec cannot write docvalues
   [junit4]   2> 		at org.apache.lucene.codecs.lucene3x.Lucene3xCodec$1.fieldsConsumer(Lucene3xCodec.java:73)
   [junit4]   2> 		at org.apache.lucene.index.DocValuesProcessor.flush(DocValuesProcessor.java:77)
   [junit4]   2> 		at org.apache.lucene.index.TwoStoredFieldsConsumers.flush(TwoStoredFieldsConsumers.java:42)
   [junit4]   2> 		at org.apache.lucene.index.DocFieldProcessor.flush(DocFieldProcessor.java:80)
   [junit4]   2> 		at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:465)
   [junit4]   2> 		at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:506)
   [junit4]   2> 		at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:616)
   [junit4]   2> 		at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:2864)
   [junit4]   2> 		at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3022)
   [junit4]   2> 		at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:2989)
   [junit4]   2> 		at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:578)
   [junit4]   2> 		at org.apache.solr.update.processor.RunUpdateProcessor.processCommit(RunUpdateProcessorFactory.java:95)
   [junit4]   2> 		at org.apache.solr.update.processor.UpdateRequestProcessor.processCommit(UpdateRequestProcessor.java:64)
   [junit4]   2> 		at org.apache.solr.update.processor.DistributedUpdateProcessor.doLocalCommit(DistributedUpdateProcessor.java:1457)
   [junit4]   2> 		at org.apache.solr.update.processor.DistributedUpdateProcessor.processCommit(DistributedUpdateProcessor.java:1434)
   [junit4]   2> 		at org.apache.solr.update.processor.LogUpdateProcessor.processCommit(LogUpdateProcessorFactory.java:157)
   [junit4]   2> 		at org.apache.solr.handler.RequestHandlerUtils.handleCommit(RequestHandlerUtils.java:69)
   [junit4]   2> 		at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:68)
   [junit4]   2> 		at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
   [junit4]   2> 		at org.apache.solr.core.SolrCore.execute(SolrCore.java:1916)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:780)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:427)
   [junit4]   2> 		at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:134)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
   [junit4]   2> 		at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
   [junit4]   2> 		at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
   [junit4]   2> 		at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
   [junit4]   2> 		at org.eclipse.jetty.server.Server.handle(Server.java:368)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
   [junit4]   2> 		at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
   [junit4]   2> 		at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:861)
   [junit4]   2> 		at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
   [junit4]   2> 		at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]   2> 		at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]   2> 		at org.eclipse.jetty.server.ssl.SslSocketConnector$SslConnectorEndPoint.run(SslSocketConnector.java:670)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 		at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 	
   [junit4]   2> 4992489 T16114 C2695 oasc.SolrException.log ERROR null:java.lang.UnsupportedOperationException: this codec cannot write docvalues
   [junit4]   2> 		at org.apache.lucene.codecs.lucene3x.Lucene3xCodec$1.fieldsConsumer(Lucene3xCodec.java:73)
   [junit4]   2> 		at org.apache.lucene.index.DocValuesProcessor.flush(DocValuesProcessor.java:77)
   [junit4]   2> 		at org.apache.lucene.index.TwoStoredFieldsConsumers.flush(TwoStoredFieldsConsumers.java:42)
   [junit4]   2> 		at org.apache.lucene.index.DocFieldProcessor.flush(DocFieldProcessor.java:80)
   [junit4]   2> 		at org.apache.lucene.index.DocumentsWriterPerThread.flush(DocumentsWriterPerThread.java:465)
   [junit4]   2> 		at org.apache.lucene.index.DocumentsWriter.doFlush(DocumentsWriter.java:506)
   [junit4]   2> 		at org.apache.lucene.index.DocumentsWriter.flushAllThreads(DocumentsWriter.java:616)
   [junit4]   2> 		at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:2864)
   [junit4]   2> 		at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3022)
   [junit4]   2> 		at org.apache.lucene.ind

[...truncated too long message...]

lang.Thread.run(Thread.java:724)
   [junit4]   2> 	  60) Thread[id=743, name=HashSessionScavenger-17, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.lang.Object.wait(Native Method)
   [junit4]   2> 	        at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]   2> 	        at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 	  61) Thread[id=744, name=qtp781588329-744 Acceptor0 SocketConnector@127.0.0.1:33558, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.net.PlainSocketImpl.socketAccept(Native Method)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398)
   [junit4]   2> 	        at java.net.ServerSocket.implAccept(ServerSocket.java:530)
   [junit4]   2> 	        at java.net.ServerSocket.accept(ServerSocket.java:498)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector.accept(SocketConnector.java:117)
   [junit4]   2> 	        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 	  62) Thread[id=627, name=qtp1527114287-627 Acceptor0 SocketConnector@127.0.0.1:13711, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.net.PlainSocketImpl.socketAccept(Native Method)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398)
   [junit4]   2> 	        at java.net.ServerSocket.implAccept(ServerSocket.java:530)
   [junit4]   2> 	        at java.net.ServerSocket.accept(ServerSocket.java:498)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector.accept(SocketConnector.java:117)
   [junit4]   2> 	        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 	  63) Thread[id=626, name=HashSessionScavenger-11, state=TIMED_WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.lang.Object.wait(Native Method)
   [junit4]   2> 	        at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]   2> 	        at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 	  64) Thread[id=718, name=searcherExecutor-396-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 	  65) Thread[id=956, name=searcherExecutor-479-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 	  66) Thread[id=637, name=TEST-ChaosMonkeySafeLeaderTest.testDistribSearch-seed#[4D75887B35EB759B]-SendThread(localhost:11714), state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.nio.ch.KQueueArrayWrapper.interrupt(Native Method)
   [junit4]   2> 	        at sun.nio.ch.KQueueArrayWrapper.interrupt(KQueueArrayWrapper.java:164)
   [junit4]   2> 	        at sun.nio.ch.KQueueSelectorImpl.wakeup(KQueueSelectorImpl.java:245)
   [junit4]   2> 	        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:210)
   [junit4]   2> 	        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:216)
   [junit4]   2> 	        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:102)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   [junit4]   2> 	  67) Thread[id=986, name=qtp1698469511-986 Acceptor0 SocketConnector@127.0.0.1:39533, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.net.PlainSocketImpl.socketAccept(Native Method)
   [junit4]   2> 	        at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:398)
   [junit4]   2> 	        at java.net.ServerSocket.implAccept(ServerSocket.java:530)
   [junit4]   2> 	        at java.net.ServerSocket.accept(ServerSocket.java:498)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector.accept(SocketConnector.java:117)
   [junit4]   2> 	        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 	  68) Thread[id=719, name=Thread-268, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.lang.Object.wait(Native Method)
   [junit4]   2> 	        at java.lang.Object.wait(Object.java:503)
   [junit4]   2> 	        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1018)
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=fi_FI, timezone=America/Caracas
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=70,free=145633128,total=321912832
   [junit4]   2> NOTE: All tests run in this JVM: [TermsComponentTest, JSONWriterTest, DeleteReplicaTest, SOLR749Test, SolrXmlInZkTest, TestReloadAndDeleteDocs, TestFieldTypeResource, RAMDirectoryFactoryTest, ParsingFieldUpdateProcessorsTest, SimpleFacetsTest, TestComponentsName, TestOmitPositions, TestIndexSearcher, TimeZoneUtilsTest, SolrCoreTest, ShowFileRequestHandlerTest, LeaderElectionIntegrationTest, TestCopyFieldCollectionResource, CursorPagingTest, TestNonDefinedSimilarityFactory, HdfsBasicDistributedZk2Test, WordBreakSolrSpellCheckerTest, ChaosMonkeySafeLeaderTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.seed=4D75887B35EB759B -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=fi_FI -Dtests.timezone=America/Caracas -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | ChaosMonkeySafeLeaderTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4D75887B35EB759B]:0)
   [junit4] Completed on J1 in 7224.58s, 1 test, 2 errors <<< FAILURES!

[...truncated 213 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:451: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:490: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1278: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:911: There were test failures: 363 suites, 1565 tests, 1 suite-level error, 2 errors, 228 ignored (7 assumptions)

Total time: 151 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1910 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1910/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:25648/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:25648/collection1
	at __randomizedtesting.SeedInfo.seed([57BED07FA9EB55CB:D6585E67DEB435F7]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:505)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:168)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:146)
	at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:663)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:150)
	at java.net.SocketInputStream.read(SocketInputStream.java:121)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
	... 47 more




Build Log:
[...truncated 10145 lines...]
   [junit4] JVM J1: stdout was not empty, see: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/temp/junit4-J1-20140225_055110_474.sysout
   [junit4] >>> JVM J1: stdout (verbatim) ----
   [junit4] #
   [junit4] # A fatal error has been detected by the Java Runtime Environment:
   [junit4] #
   [junit4] #  Internal Error (assembler_x86.inline.hpp:46), pid=3414, tid=34378089472
   [junit4] #  guarantee(this->is8bit(imm8)) failed: Short forward jump exceeds 8-bit offset
   [junit4] #
   [junit4] # JRE version: 7.0-b15
   [junit4] # Java VM: OpenJDK 64-Bit Server VM (23.21-b01 mixed mode bsd-amd64 compressed oops)
   [junit4] # Core dump written. Default location: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/core or core.3414
   [junit4] #
   [junit4] # An error report file with more information is saved as:
   [junit4] # /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/hs_err_pid3414.log
   [junit4] #
   [junit4] # If you would like to submit a bug report, please visit:
   [junit4] #   http://bugreport.sun.com/bugreport/crash.jsp
   [junit4] #
   [junit4] <<< JVM J1: EOF ----

[...truncated 44 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> 1860106 T2434 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 1860106 T2434 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1860113 T2434 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1393309333886
   [junit4]   2> 1860115 T2434 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1860115 T2435 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1860216 T2434 oasc.ZkTestServer.run start zk server on port:56655
   [junit4]   2> 1860217 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1860222 T2441 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c4d0ac5 name:ZooKeeperConnection Watcher:127.0.0.1:56655 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1860222 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1860223 T2434 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1860248 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1860251 T2443 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10e50619 name:ZooKeeperConnection Watcher:127.0.0.1:56655/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1860251 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1860251 T2434 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1860257 T2434 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1860260 T2434 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1860269 T2434 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1860273 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1860274 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1860288 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1860289 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1860395 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1860395 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1860399 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1860400 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1860403 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1860404 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1860408 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1860409 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1860412 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1860413 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1860416 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1860417 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1860420 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1860421 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1860424 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1860425 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1860428 T2434 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1860429 T2434 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1860434 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1860442 T2445 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@467f0c23 name:ZooKeeperConnection Watcher:127.0.0.1:56655/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1860443 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1861424 T2434 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1861428 T2434 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1861430 T2434 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:41824
   [junit4]   2> 1861432 T2434 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1861433 T2434 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1861433 T2434 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218
   [junit4]   2> 1861434 T2434 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218/'
   [junit4]   2> 1861459 T2434 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218/solr.xml
   [junit4]   2> 1861525 T2434 oasc.CoreContainer.<init> New CoreContainer 721619428
   [junit4]   2> 1861526 T2434 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218/]
   [junit4]   2> 1861528 T2434 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1861528 T2434 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1861528 T2434 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1861528 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1861529 T2434 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1861529 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1861529 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1861530 T2434 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1861530 T2434 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1861541 T2434 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1861541 T2434 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1861541 T2434 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1861542 T2434 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56655/solr
   [junit4]   2> 1861542 T2434 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1861543 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1861546 T2456 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f2b53dd name:ZooKeeperConnection Watcher:127.0.0.1:56655 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1861547 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1861550 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1861565 T2458 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a294371 name:ZooKeeperConnection Watcher:127.0.0.1:56655/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1861565 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1861568 T2434 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1861573 T2434 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1861582 T2434 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1861585 T2434 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41824_
   [junit4]   2> 1861587 T2434 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41824_
   [junit4]   2> 1861591 T2434 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1861594 T2434 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1861608 T2434 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:41824_
   [junit4]   2> 1861608 T2434 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1861611 T2434 oasc.Overseer.start Overseer (id=91311920506011652-127.0.0.1:41824_-n_0000000000) starting
   [junit4]   2> 1861620 T2434 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1861632 T2460 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1861633 T2434 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1861633 T2460 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 1861635 T2434 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1861643 T2434 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1861645 T2459 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1861652 T2461 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1861653 T2461 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1861654 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1861654 T2461 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1861655 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1861656 T2459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:41824",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41824_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1861657 T2459 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1861657 T2459 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1861660 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1861668 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1862656 T2461 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1862656 T2461 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218/collection1
   [junit4]   2> 1862656 T2461 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1862657 T2461 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1862658 T2461 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1862659 T2461 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1862659 T2461 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218/collection1/'
   [junit4]   2> 1862661 T2461 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218/collection1/lib/.svn/' to classloader
   [junit4]   2> 1862661 T2461 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218/collection1/lib/classes/' to classloader
   [junit4]   2> 1862662 T2461 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218/collection1/lib/README' to classloader
   [junit4]   2> 1862736 T2461 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 1862809 T2461 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1862911 T2461 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1862923 T2461 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1863582 T2461 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1863583 T2461 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1863584 T2461 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1863597 T2461 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1863602 T2461 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1863619 T2461 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1863625 T2461 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1863629 T2461 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1863631 T2461 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1863631 T2461 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1863631 T2461 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1863633 T2461 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1863633 T2461 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1863633 T2461 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1863634 T2461 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1393309334218/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/control/data/
   [junit4]   2> 1863634 T2461 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dbbeee7
   [junit4]   2> 1863635 T2461 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/control/data
   [junit4]   2> 1863635 T2461 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/control/data/index/
   [junit4]   2> 1863635 T2461 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1863636 T2461 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/control/data/index
   [junit4]   2> 1863636 T2461 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1045610588, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1863637 T2461 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6768fcfb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7787363d),segFN=segments_1,generation=1}
   [junit4]   2> 1863638 T2461 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1863642 T2461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1863643 T2461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1863643 T2461 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1863643 T2461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1863644 T2461 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1863644 T2461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1863645 T2461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1863645 T2461 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1863645 T2461 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1863646 T2461 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1863647 T2461 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1863647 T2461 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1863647 T2461 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1863648 T2461 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1863648 T2461 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1863649 T2461 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1863665 T2461 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1863672 T2461 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1863672 T2461 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1863673 T2461 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.793231173948885]
   [junit4]   2> 1863673 T2461 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6768fcfb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7787363d),segFN=segments_1,generation=1}
   [junit4]   2> 1863674 T2461 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1863674 T2461 oass.SolrIndexSearcher.<init> Opening Searcher@29b7a602[collection1] main
   [junit4]   2> 1863678 T2462 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@29b7a602[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1863680 T2461 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1863681 T2434 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1863681 T2434 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1863681 T2465 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:41824 collection:control_collection shard:shard1
   [junit4]   2> 1863684 T2465 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1863687 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1863689 T2467 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3acbaef9 name:ZooKeeperConnection Watcher:127.0.0.1:56655/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1863689 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1863704 T2434 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1863708 T2465 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1863708 T2434 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1863711 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1863711 T2465 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1863711 T2465 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C412 name=collection1 org.apache.solr.core.SolrCore@2610515e url=https://127.0.0.1:41824/collection1 node=127.0.0.1:41824_ C412_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:41824, core=collection1, node_name=127.0.0.1:41824_}
   [junit4]   2> 1863712 T2465 C412 P41824 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:41824/collection1/
   [junit4]   2> 1863712 T2465 C412 P41824 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1863712 T2465 C412 P41824 oasc.SyncStrategy.syncToMe https://127.0.0.1:41824/collection1/ has no replicas
   [junit4]   2> 1863712 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1863713 T2465 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:41824/collection1/ shard1
   [junit4]   2> 1863713 T2465 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1863722 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1863723 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1863723 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1863742 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1863746 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1863849 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1863849 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1863895 T2465 oasc.ZkController.register We are https://127.0.0.1:41824/collection1/ and leader is https://127.0.0.1:41824/collection1/
   [junit4]   2> 1863895 T2465 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:41824
   [junit4]   2> 1863895 T2465 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1863895 T2465 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1863895 T2465 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1863897 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1863897 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1863898 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1863898 T2465 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1863899 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1863900 T2459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:41824",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41824_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1863909 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1864014 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1864014 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1864722 T2434 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1864723 T2434 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1864727 T2434 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1864728 T2434 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:25648
   [junit4]   2> 1864730 T2434 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1864731 T2434 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1864731 T2434 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481
   [junit4]   2> 1864731 T2434 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481/'
   [junit4]   2> 1864757 T2434 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481/solr.xml
   [junit4]   2> 1864851 T2434 oasc.CoreContainer.<init> New CoreContainer 272096330
   [junit4]   2> 1864852 T2434 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481/]
   [junit4]   2> 1864854 T2434 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1864854 T2434 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1864855 T2434 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1864855 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1864856 T2434 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1864856 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1864857 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1864857 T2434 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1864858 T2434 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1864873 T2434 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1864874 T2434 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1864874 T2434 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1864875 T2434 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56655/solr
   [junit4]   2> 1864875 T2434 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1864876 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1864879 T2478 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5a4bd3e name:ZooKeeperConnection Watcher:127.0.0.1:56655 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1864880 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1864899 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1864901 T2480 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b9c6122 name:ZooKeeperConnection Watcher:127.0.0.1:56655/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1864901 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1864908 T2434 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1865911 T2434 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25648_
   [junit4]   2> 1865914 T2434 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25648_
   [junit4]   2> 1865918 T2458 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1865918 T2480 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1865918 T2467 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1865929 T2481 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1865929 T2481 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1865931 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1865931 T2481 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1865931 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1865931 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1865932 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1865933 T2459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:25648",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25648_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1865934 T2459 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1865934 T2459 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1865938 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1865939 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1865939 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1865939 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1866932 T2481 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1866932 T2481 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481/collection1
   [junit4]   2> 1866932 T2481 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1866934 T2481 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1866934 T2481 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1866935 T2481 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1866935 T2481 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481/collection1/'
   [junit4]   2> 1866937 T2481 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481/collection1/lib/.svn/' to classloader
   [junit4]   2> 1866937 T2481 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481/collection1/lib/README' to classloader
   [junit4]   2> 1866938 T2481 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481/collection1/lib/classes/' to classloader
   [junit4]   2> 1867011 T2481 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 1867085 T2481 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1867188 T2481 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1867200 T2481 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1867858 T2481 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1867859 T2481 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1867860 T2481 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1867874 T2481 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1867878 T2481 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1867896 T2481 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1867901 T2481 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1867907 T2481 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1867908 T2481 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1867909 T2481 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1867909 T2481 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1867910 T2481 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1867911 T2481 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1867911 T2481 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1867911 T2481 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1393309337481/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty1/
   [junit4]   2> 1867912 T2481 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dbbeee7
   [junit4]   2> 1867913 T2481 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty1
   [junit4]   2> 1867913 T2481 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty1/index/
   [junit4]   2> 1867913 T2481 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1867914 T2481 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty1/index
   [junit4]   2> 1867914 T2481 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1045610588, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1867915 T2481 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e48b3db lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c1a71ac),segFN=segments_1,generation=1}
   [junit4]   2> 1867916 T2481 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1867921 T2481 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1867922 T2481 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1867922 T2481 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1867923 T2481 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1867923 T2481 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1867923 T2481 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1867924 T2481 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1867924 T2481 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1867924 T2481 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1867925 T2481 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1867926 T2481 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1867926 T2481 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1867927 T2481 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1867927 T2481 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1867928 T2481 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1867929 T2481 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1867945 T2481 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1867951 T2481 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1867951 T2481 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1867952 T2481 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.793231173948885]
   [junit4]   2> 1867953 T2481 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4e48b3db lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2c1a71ac),segFN=segments_1,generation=1}
   [junit4]   2> 1867953 T2481 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1867954 T2481 oass.SolrIndexSearcher.<init> Opening Searcher@a4522e7[collection1] main
   [junit4]   2> 1867959 T2482 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a4522e7[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1867962 T2481 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1867963 T2434 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1867964 T2434 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1867963 T2485 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:25648 collection:collection1 shard:shard2
   [junit4]   2> 1867966 T2485 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1868010 T2485 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1868019 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1868019 T2485 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1868020 T2485 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C413 name=collection1 org.apache.solr.core.SolrCore@6c575b29 url=https://127.0.0.1:25648/collection1 node=127.0.0.1:25648_ C413_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:25648, core=collection1, node_name=127.0.0.1:25648_}
   [junit4]   2> 1868020 T2485 C413 P25648 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:25648/collection1/
   [junit4]   2> 1868020 T2485 C413 P25648 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1868021 T2485 C413 P25648 oasc.SyncStrategy.syncToMe https://127.0.0.1:25648/collection1/ has no replicas
   [junit4]   2> 1868021 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1868021 T2485 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:25648/collection1/ shard2
   [junit4]   2> 1868021 T2485 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1868034 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1868036 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1868036 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1868036 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1868041 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1868046 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1868150 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1868150 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1868150 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1868193 T2485 oasc.ZkController.register We are https://127.0.0.1:25648/collection1/ and leader is https://127.0.0.1:25648/collection1/
   [junit4]   2> 1868193 T2485 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:25648
   [junit4]   2> 1868193 T2485 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1868193 T2485 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1868193 T2485 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1868195 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1868195 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1868195 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1868196 T2485 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1868197 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1868198 T2459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:25648",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25648_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1868207 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1868310 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1868310 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1868310 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1868915 T2434 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1868916 T2434 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1868920 T2434 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1868921 T2434 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:46948
   [junit4]   2> 1868923 T2434 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1868924 T2434 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1868924 T2434 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739
   [junit4]   2> 1868924 T2434 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739/'
   [junit4]   2> 1868950 T2434 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739/solr.xml
   [junit4]   2> 1869015 T2434 oasc.CoreContainer.<init> New CoreContainer 244843597
   [junit4]   2> 1869016 T2434 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739/]
   [junit4]   2> 1869017 T2434 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1869017 T2434 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1869018 T2434 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1869018 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1869018 T2434 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1869018 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1869019 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1869019 T2434 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1869019 T2434 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1869030 T2434 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1869030 T2434 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1869030 T2434 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1869031 T2434 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56655/solr
   [junit4]   2> 1869031 T2434 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1869032 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1869061 T2496 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62c8a780 name:ZooKeeperConnection Watcher:127.0.0.1:56655 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1869061 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1869071 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1869073 T2498 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5579b45f name:ZooKeeperConnection Watcher:127.0.0.1:56655/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1869073 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1869080 T2434 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1870083 T2434 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46948_
   [junit4]   2> 1870094 T2434 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:46948_
   [junit4]   2> 1870098 T2467 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1870098 T2458 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1870099 T2498 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1870098 T2480 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1870116 T2499 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1870116 T2499 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1870118 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1870118 T2499 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1870118 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1870118 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1870120 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1870121 T2459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:46948",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:46948_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1870121 T2459 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1870121 T2459 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1870125 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1870127 T2498 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1870127 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1870127 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1870127 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1871119 T2499 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1871119 T2499 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739/collection1
   [junit4]   2> 1871119 T2499 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1871120 T2499 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1871120 T2499 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1871122 T2499 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1871122 T2499 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739/collection1/'
   [junit4]   2> 1871123 T2499 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739/collection1/lib/.svn/' to classloader
   [junit4]   2> 1871124 T2499 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739/collection1/lib/classes/' to classloader
   [junit4]   2> 1871125 T2499 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739/collection1/lib/README' to classloader
   [junit4]   2> 1871197 T2499 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 1871270 T2499 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1871372 T2499 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1871383 T2499 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1872049 T2499 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1872051 T2499 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1872051 T2499 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1872065 T2499 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1872070 T2499 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1872090 T2499 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1872095 T2499 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1872100 T2499 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1872102 T2499 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1872102 T2499 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1872102 T2499 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1872103 T2499 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1872104 T2499 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1872104 T2499 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1872104 T2499 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1393309341739/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty2/
   [junit4]   2> 1872105 T2499 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6dbbeee7
   [junit4]   2> 1872106 T2499 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty2
   [junit4]   2> 1872106 T2499 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty2/index/
   [junit4]   2> 1872107 T2499 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1872107 T2499 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1393309333886/jetty2/index
   [junit4]   2> 1872108 T2499 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1045610588, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1872109 T2499 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@9a93c33 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@14a81561),segFN=segments_1,generation=1}
   [junit4]   2> 1872109 T2499 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1872115 T2499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1872115 T2499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1872115 T2499 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1872116 T2499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1872116 T2499 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1872116 T2499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1872117 T2499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1872117 T2499 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1872118 T2499 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1872132 T2499 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1872133 T2499 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1872133 T2499 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1872134 T2499 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1872135 T2499 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1872135 T2499 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1872136 T2499 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1872153 T2499 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1872158 T2499 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1872159 T2499 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1872159 T2499 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=24, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.793231173948885]
   [junit4]   2> 1872160 T2499 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@9a93c33 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@14a81561),segFN=segments_1,generation=1}
   [junit4]   2> 1872161 T2499 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1872161 T2499 oass.SolrIndexSearcher.<init> Opening Searcher@3cd58ba7[collection1] main
   [junit4]   2> 1872166 T2500 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3cd58ba7[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1872169 T2499 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1872170 T2434 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 1872171 T2434 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1872170 T2503 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:46948 collection:collection1 shard:shard1
   [junit4]   2> 1872173 T2503 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1872202 T2503 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1872213 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1872214 T2503 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1872214 T2503 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C414 name=collection1 org.apache.solr.core.SolrCore@3094b691 url=https://127.0.0.1:46948/collection1 node=127.0.0.1:46948_ C414_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:46948, core=collection1, node_name=127.0.0.1:46948_}
   [junit4]   2> 1872214 T2503 C414 P46948 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:46948/collection1/
   [junit4]   2> 1872214 T2503 C414 P46948 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1872215 T2503 C414 P46948 oasc.SyncStrategy.syncToMe https://127.0.0.1:46948/collection1/ has no replicas
   [junit4]   2> 1872215 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1872215 T2503 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:46948/collection1/ shard1
   [junit4]   2> 1872215 T2503 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1872219 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1872229 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872229 T2498 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872229 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872229 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872240 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1872244 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1872351 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872351 T2498 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872351 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872351 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872392 T2503 oasc.ZkController.register We are https://127.0.0.1:46948/collection1/ and leader is https://127.0.0.1:46948/collection1/
   [junit4]   2> 1872392 T2503 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:46948
   [junit4]   2> 1872392 T2503 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1872392 T2503 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1872392 T2503 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1872394 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1872394 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1872394 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1872395 T2503 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1872396 T2459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1872397 T2459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:46948",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:46948_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1872400 T2458 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1872507 T2480 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872507 T2458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872507 T2467 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1872507 T2498 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1873046 T2434 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 1873048 T2434 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1873052 T2434 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1873053 T2434 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:61460
   [junit4]   2> 1873056 T2434 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1873056 T2434 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1873056 T2434 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1393309345946
   [junit4]   2> 1873057 T2434 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1393309345946/'
   [junit4]   2> 1873082 T2434 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1393309345946/solr.xml
   [junit4]   2> 1873148 T2434 oasc.CoreContainer.<init> New CoreContainer 1850918236
   [junit4]   2> 1873149 T2434 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1393309345946/]
   [junit4]   2> 1873150 T2434 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1873150 T2434 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1873150 T2434 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1873151 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1873151 T2434 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1873151 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1873152 T2434 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1873152 T2434 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1873152 T2434 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1873161 T2434 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1873162 T2434 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1873162 T2434 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1873163 T2434 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56655/solr
   [junit4]   2> 1873163 T2434 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1873164 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1873194 T2514 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@227fcf6b name:ZooKeeperConnection Watcher:127.0.0.1:56655 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1873194 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1873198 T2434 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1873207 T2516 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8cc558e name:ZooKeeperConnection Watcher:127.0.0.1:56655/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1873208 T2434 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1873214 T2434 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1874218 T2434 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:61460_
   [junit4]   2> 1874227 T2434 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.

[...truncated too long message...]

r:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/lib/spatial4j-0.4.1.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/commons-io-2.1.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/httpclient-4.3.1.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/httpcore-4.3.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/httpmime-4.3.1.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/jcl-over-slf4j-1.7.6.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/jul-to-slf4j-1.7.6.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/log4j-1.2.16.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/noggit-0.5.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/slf4j-api-1.7.6.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/slf4j-log4j12-1.7.6.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/wstx-asl-3.2.7.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/solrj/lib/zookeeper-3.4.5.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-continuation-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-deploy-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-http-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-io-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-jmx-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-security-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-server-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-servlet-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-util-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-webapp-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/jetty-xml-8.1.10.v20130312.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/servlet-api-3.0.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/example-DIH/solr/db/lib/derby-10.9.1.0.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/example-DIH/solr/db/lib/hsqldb-1.8.0.10.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/build/core/classes/java:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/test-framework/lib/junit-4.10.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/test-framework/lib/randomizedtesting-runner-2.0.13.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/antlr-runtime-3.5.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/asm-4.1.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/asm-commons-4.1.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/cglib-nodep-2.2.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/commons-collections-3.2.1.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/dom4j-1.6.1.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/easymock-3.0.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/hadoop-common-2.2.0-tests.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/hadoop-hdfs-2.2.0-tests.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/hppc-0.5.2.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/javax.servlet-api-3.0.1.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/jersey-core-1.8.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/jetty-6.1.26.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/jetty-util-6.1.26.jar:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/test-lib/objenesis-1.2.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-launcher.jar:/home/hudson/.ant/lib/ivy-2.3.0.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-commons-logging.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-regexp.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-oro.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-swing.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jdepend.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-xalan2.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-bsf.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-junit4.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-commons-net.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-antlr.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-junit.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jmf.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-netrexx.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-log4j.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-resolver.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jsch.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-testutil.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-apache-bcel.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-jai.jar:/home/hudson/hudson-slave/tools/hudson.tasks.Ant_AntInstallation/ant-1.8.2/lib/ant-javamail.jar:/usr/local/openjdk7/lib/tools.jar:/home/hudson/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.0.13.jar -ea:org.apache.lucene... -ea:org.apache.solr... com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/temp/junit4-J1-20140225_055110_474.events @/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/temp/junit4-J1-20140225_055110_474.suites
   [junit4] ERROR: JVM J1 ended with an exception: Forked process returned with error code: 134. Very likely a JVM crash.  Process output piped in logs above.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1373)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:119)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:911)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:908)
   [junit4] 	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:166)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4] 	at java.lang.Thread.run(Thread.java:724)

BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:471: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:451: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:490: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1278: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:911: At least one slave process threw an exception, first: Forked process returned with error code: 134. Very likely a JVM crash.  Process output piped in logs above.

Total time: 105 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1909 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1909/

All tests passed

Build Log:
[...truncated 28657 lines...]
check-licenses:
     [echo] License check under: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jcl-over-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jcl-over-slf4j-1.6.6.jar.sha1

 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/jul-to-slf4j-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/jul-to-slf4j-1.6.6.jar.sha1
 [licenses] MISSING sha1 checksum file for: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/example/lib/ext/slf4j-api-1.6.6.jar
 [licenses] EXPECTED sha1 checksum file : /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/licenses/slf4j-api-1.6.6.jar.sha1

[...truncated 3 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:64: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:254: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/tools/custom-tasks.xml:62: License check failed. Check the logs.

Total time: 108 minutes 22 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1908 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1908/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZk2Test.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:25285/collection1, https://127.0.0.1:44407/collection1]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:25285/collection1, https://127.0.0.1:44407/collection1]
	at __randomizedtesting.SeedInfo.seed([EB0298211B0AB49E:6AE416396C55D4A2]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:387)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:635)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.queryServer(AbstractFullDistribZkTestBase.java:1427)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:563)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:545)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:524)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:296)
	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:113)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: no servers hosting shard: 
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:495)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:302)
	... 50 more




Build Log:
[...truncated 10523 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZk2Test
   [junit4]   2> 2138169 T18218 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 2138169 T18218 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 2138175 T18218 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solrtest-BasicDistributedZk2Test-1393071038017
   [junit4]   2> 2138177 T18218 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 2138178 T18219 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2138279 T18218 oasc.ZkTestServer.run start zk server on port:63284
   [junit4]   2> 2138280 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2138284 T18225 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62ed6942 name:ZooKeeperConnection Watcher:127.0.0.1:63284 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2138285 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2138285 T18218 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2138307 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2138309 T18227 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61e5efcf name:ZooKeeperConnection Watcher:127.0.0.1:63284/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2138310 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2138310 T18218 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2138324 T18218 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2138326 T18218 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2138328 T18218 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2138338 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2138339 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2138352 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2138353 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2138456 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2138457 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2138460 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2138460 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2138462 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2138463 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2138474 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2138475 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2138477 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2138478 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 2138480 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2138480 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2138489 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2138490 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2138493 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2138494 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2138497 T18218 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2138497 T18218 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2138501 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2138511 T18229 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f05941b name:ZooKeeperConnection Watcher:127.0.0.1:63284/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2138511 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2138744 T18218 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2138748 T18218 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2138755 T18218 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:56182
   [junit4]   2> 2138758 T18218 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2138759 T18218 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2138759 T18218 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1393071038354
   [junit4]   2> 2138759 T18218 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1393071038354/'
   [junit4]   2> 2138792 T18218 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1393071038354/solr.xml
   [junit4]   2> 2138878 T18218 oasc.CoreContainer.<init> New CoreContainer 547885810
   [junit4]   2> 2138879 T18218 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1393071038354/]
   [junit4]   2> 2138881 T18218 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2138881 T18218 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2138882 T18218 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2138882 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2138883 T18218 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2138883 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2138883 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2138884 T18218 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2138884 T18218 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2138897 T18218 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2138898 T18218 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2138898 T18218 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2138899 T18218 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63284/solr
   [junit4]   2> 2138899 T18218 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2138900 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2138903 T18240 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@349c961d name:ZooKeeperConnection Watcher:127.0.0.1:63284 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2138904 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2138918 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2138920 T18242 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@63cec624 name:ZooKeeperConnection Watcher:127.0.0.1:63284/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2138920 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2138927 T18218 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2138931 T18218 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2138939 T18218 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2138946 T18218 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56182_
   [junit4]   2> 2138948 T18218 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56182_
   [junit4]   2> 2138964 T18218 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 2138966 T18218 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2138979 T18218 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:56182_
   [junit4]   2> 2138979 T18218 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2138982 T18218 oasc.Overseer.start Overseer (id=91296303547875332-127.0.0.1:56182_-n_0000000000) starting
   [junit4]   2> 2138994 T18218 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2139017 T18244 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2139018 T18218 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2139018 T18244 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2139020 T18218 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2139021 T18218 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2139023 T18243 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2139027 T18245 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2139028 T18245 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2139029 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2139029 T18245 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2139030 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2139030 T18243 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:56182",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56182_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2139030 T18243 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 2139030 T18243 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2139032 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2139042 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2140030 T18245 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2140030 T18245 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1393071038354/collection1
   [junit4]   2> 2140031 T18245 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 2140031 T18245 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2140032 T18245 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2140033 T18245 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2140033 T18245 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1393071038354/collection1/'
   [junit4]   2> 2140035 T18245 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1393071038354/collection1/lib/README' to classloader
   [junit4]   2> 2140035 T18245 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1393071038354/collection1/lib/classes/' to classloader
   [junit4]   2> 2140110 T18245 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 2140189 T18245 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2140291 T18245 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2140304 T18245 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2141125 T18245 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2141126 T18245 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2141127 T18245 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2141142 T18245 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2141149 T18245 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2141172 T18245 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2141179 T18245 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2141185 T18245 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2141186 T18245 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2141187 T18245 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2141187 T18245 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2141188 T18245 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2141188 T18245 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2141189 T18245 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2141189 T18245 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1393071038354/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/control/data/
   [junit4]   2> 2141189 T18245 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@175e112d
   [junit4]   2> 2141191 T18245 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/control/data
   [junit4]   2> 2141191 T18245 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/control/data/index/
   [junit4]   2> 2141191 T18245 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2141192 T18245 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/control/data/index
   [junit4]   2> 2141193 T18245 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=18.4736328125, floorSegmentMB=2.13671875, forceMergeDeletesPctAllowed=24.38549352231635, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.41506504242269715
   [junit4]   2> 2141194 T18245 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ddeecdc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@d637136),segFN=segments_1,generation=1}
   [junit4]   2> 2141195 T18245 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2141199 T18245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2141200 T18245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2141200 T18245 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2141201 T18245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2141201 T18245 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2141201 T18245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2141202 T18245 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2141203 T18245 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2141203 T18245 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2141204 T18245 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2141205 T18245 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2141205 T18245 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2141205 T18245 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2141206 T18245 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2141206 T18245 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2141207 T18245 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2141227 T18245 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2141235 T18245 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2141236 T18245 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2141237 T18245 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=36, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.20957049980281273]
   [junit4]   2> 2141237 T18245 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6ddeecdc lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@d637136),segFN=segments_1,generation=1}
   [junit4]   2> 2141238 T18245 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2141238 T18245 oass.SolrIndexSearcher.<init> Opening Searcher@149b9d2b[collection1] main
   [junit4]   2> 2141242 T18246 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@149b9d2b[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2141244 T18245 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2141245 T18218 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 2141245 T18218 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2141245 T18249 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:56182 collection:control_collection shard:shard1
   [junit4]   2> 2141247 T18249 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 2141249 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2141264 T18251 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@492d8c0f name:ZooKeeperConnection Watcher:127.0.0.1:63284/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2141264 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2141266 T18218 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2141267 T18249 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2141269 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2141269 T18218 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2141269 T18249 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2141270 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2141270 T18249 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3427 name=collection1 org.apache.solr.core.SolrCore@2ea08756 url=https://127.0.0.1:56182/collection1 node=127.0.0.1:56182_ C3427_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:56182, core=collection1, node_name=127.0.0.1:56182_}
   [junit4]   2> 2141270 T18249 C3427 P56182 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:56182/collection1/
   [junit4]   2> 2141271 T18249 C3427 P56182 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2141271 T18249 C3427 P56182 oasc.SyncStrategy.syncToMe https://127.0.0.1:56182/collection1/ has no replicas
   [junit4]   2> 2141271 T18249 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:56182/collection1/ shard1
   [junit4]   2> 2141272 T18249 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 2141272 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2141273 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2141273 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2141278 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2141279 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2141281 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2141385 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2141385 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2141431 T18249 oasc.ZkController.register We are https://127.0.0.1:56182/collection1/ and leader is https://127.0.0.1:56182/collection1/
   [junit4]   2> 2141431 T18249 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:56182
   [junit4]   2> 2141431 T18249 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2141432 T18249 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 2141432 T18249 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2141434 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2141434 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2141434 T18249 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2141434 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2141435 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2141436 T18243 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:56182",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56182_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 2141438 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2141483 T18218 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 2141485 T18218 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2141489 T18218 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2141491 T18218 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:44407
   [junit4]   2> 2141493 T18218 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2141494 T18218 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2141494 T18218 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1393071041110
   [junit4]   2> 2141494 T18218 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1393071041110/'
   [junit4]   2> 2141530 T18218 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1393071041110/solr.xml
   [junit4]   2> 2141541 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2141541 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2141613 T18218 oasc.CoreContainer.<init> New CoreContainer 1675787702
   [junit4]   2> 2141613 T18218 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1393071041110/]
   [junit4]   2> 2141615 T18218 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2141615 T18218 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2141616 T18218 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2141616 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2141616 T18218 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2141617 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2141617 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2141617 T18218 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2141618 T18218 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2141628 T18218 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2141629 T18218 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2141629 T18218 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2141630 T18218 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63284/solr
   [junit4]   2> 2141630 T18218 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2141640 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2141644 T18262 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3dee0f1c name:ZooKeeperConnection Watcher:127.0.0.1:63284 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2141646 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2141722 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2141725 T18264 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@53b9b349 name:ZooKeeperConnection Watcher:127.0.0.1:63284/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2141725 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2141731 T18218 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2142735 T18218 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44407_
   [junit4]   2> 2142762 T18218 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44407_
   [junit4]   2> 2142765 T18251 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2142765 T18264 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2142765 T18242 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2142784 T18265 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2142784 T18265 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2142786 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2142786 T18265 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2142786 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2142787 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2142787 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2142788 T18243 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:44407",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:44407_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2142788 T18243 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 2142788 T18243 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 2142791 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2142895 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2142895 T18264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2142895 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2143787 T18265 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2143788 T18265 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1393071041110/collection1
   [junit4]   2> 2143788 T18265 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2143789 T18265 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2143789 T18265 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2143790 T18265 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2143790 T18265 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1393071041110/collection1/'
   [junit4]   2> 2143791 T18265 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1393071041110/collection1/lib/README' to classloader
   [junit4]   2> 2143792 T18265 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1393071041110/collection1/lib/classes/' to classloader
   [junit4]   2> 2143864 T18265 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 2143982 T18265 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2144084 T18265 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2144099 T18265 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2144859 T18265 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2144861 T18265 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2144862 T18265 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2144877 T18265 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2144882 T18265 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2144896 T18265 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2144902 T18265 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2144908 T18265 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2144910 T18265 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2144910 T18265 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2144910 T18265 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2144911 T18265 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2144912 T18265 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2144912 T18265 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2144913 T18265 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1393071041110/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty1/
   [junit4]   2> 2144941 T18265 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@175e112d
   [junit4]   2> 2144942 T18265 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty1
   [junit4]   2> 2144942 T18265 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty1/index/
   [junit4]   2> 2144942 T18265 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 2144943 T18265 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty1/index
   [junit4]   2> 2144943 T18265 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=18.4736328125, floorSegmentMB=2.13671875, forceMergeDeletesPctAllowed=24.38549352231635, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.41506504242269715
   [junit4]   2> 2144944 T18265 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e3338de lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1241cd8),segFN=segments_1,generation=1}
   [junit4]   2> 2144944 T18265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2144948 T18265 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2144948 T18265 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2144949 T18265 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2144949 T18265 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2144949 T18265 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2144949 T18265 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2144950 T18265 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2144950 T18265 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2144951 T18265 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2144951 T18265 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2144952 T18265 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2144952 T18265 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2144952 T18265 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2144953 T18265 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2144953 T18265 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2144954 T18265 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2144968 T18265 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2144973 T18265 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2144973 T18265 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2144974 T18265 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=36, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.20957049980281273]
   [junit4]   2> 2144975 T18265 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1e3338de lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1241cd8),segFN=segments_1,generation=1}
   [junit4]   2> 2144975 T18265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2144975 T18265 oass.SolrIndexSearcher.<init> Opening Searcher@235b0205[collection1] main
   [junit4]   2> 2144978 T18266 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@235b0205[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2144980 T18265 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2144981 T18218 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 2144981 T18218 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2144981 T18269 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:44407 collection:collection1 shard:shard2
   [junit4]   2> 2144983 T18269 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 2145004 T18269 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 2145006 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145007 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145007 T18269 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2145007 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145007 T18269 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3428 name=collection1 org.apache.solr.core.SolrCore@2e888c60 url=https://127.0.0.1:44407/collection1 node=127.0.0.1:44407_ C3428_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:44407, core=collection1, node_name=127.0.0.1:44407_}
   [junit4]   2> 2145008 T18269 C3428 P44407 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:44407/collection1/
   [junit4]   2> 2145008 T18269 C3428 P44407 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2145008 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2145009 T18269 C3428 P44407 oasc.SyncStrategy.syncToMe https://127.0.0.1:44407/collection1/ has no replicas
   [junit4]   2> 2145009 T18269 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:44407/collection1/ shard2
   [junit4]   2> 2145009 T18269 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 2145011 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145012 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2145012 T18264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2145012 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2145015 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145016 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2145018 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145120 T18264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2145121 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2145121 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2145169 T18269 oasc.ZkController.register We are https://127.0.0.1:44407/collection1/ and leader is https://127.0.0.1:44407/collection1/
   [junit4]   2> 2145169 T18269 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:44407
   [junit4]   2> 2145169 T18269 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2145170 T18269 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 2145170 T18269 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2145172 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145172 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145172 T18269 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2145172 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145174 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2145174 T18243 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:44407",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:44407_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 2145183 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2145217 T18218 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 2145218 T18218 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2145221 T18218 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2145223 T18218 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:18179
   [junit4]   2> 2145225 T18218 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2145226 T18218 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2145226 T18218 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1393071044824
   [junit4]   2> 2145227 T18218 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1393071044824/'
   [junit4]   2> 2145261 T18218 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1393071044824/solr.xml
   [junit4]   2> 2145286 T18264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2145286 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2145286 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2145339 T18218 oasc.CoreContainer.<init> New CoreContainer 1799140997
   [junit4]   2> 2145340 T18218 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1393071044824/]
   [junit4]   2> 2145342 T18218 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2145342 T18218 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2145343 T18218 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2145343 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2145343 T18218 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2145344 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2145344 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2145344 T18218 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2145345 T18218 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2145355 T18218 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2145356 T18218 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2145356 T18218 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2145356 T18218 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63284/solr
   [junit4]   2> 2145357 T18218 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2145358 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2145360 T18280 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57e4aa7c name:ZooKeeperConnection Watcher:127.0.0.1:63284 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2145361 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2145364 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2145372 T18282 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@338e018f name:ZooKeeperConnection Watcher:127.0.0.1:63284/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2145372 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2145416 T18218 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2146420 T18218 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18179_
   [junit4]   2> 2146431 T18218 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18179_
   [junit4]   2> 2146434 T18242 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2146434 T18282 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2146434 T18264 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2146434 T18251 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2146444 T18283 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2146445 T18283 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2146447 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2146447 T18283 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2146447 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2146448 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2146448 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2146449 T18243 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:18179",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18179_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2146449 T18243 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 2146449 T18243 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2146452 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2146563 T18264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2146563 T18282 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2146563 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2146563 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2147448 T18283 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2147448 T18283 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1393071044824/collection1
   [junit4]   2> 2147449 T18283 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2147449 T18283 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2147450 T18283 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2147451 T18283 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2147451 T18283 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1393071044824/collection1/'
   [junit4]   2> 2147454 T18283 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1393071044824/collection1/lib/README' to classloader
   [junit4]   2> 2147455 T18283 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1393071044824/collection1/lib/classes/' to classloader
   [junit4]   2> 2147538 T18283 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 2147619 T18283 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2147724 T18283 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2147737 T18283 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2148553 T18283 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2148554 T18283 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2148555 T18283 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2148572 T18283 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2148578 T18283 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2148593 T18283 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2148599 T18283 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2148604 T18283 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2148605 T18283 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2148605 T18283 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2148605 T18283 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2148606 T18283 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2148607 T18283 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2148607 T18283 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2148608 T18283 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1393071044824/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty2/
   [junit4]   2> 2148608 T18283 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@175e112d
   [junit4]   2> 2148609 T18283 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty2
   [junit4]   2> 2148609 T18283 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty2/index/
   [junit4]   2> 2148610 T18283 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 2148610 T18283 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty2/index
   [junit4]   2> 2148611 T18283 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=18.4736328125, floorSegmentMB=2.13671875, forceMergeDeletesPctAllowed=24.38549352231635, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.41506504242269715
   [junit4]   2> 2148612 T18283 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@121e14e9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71cbd4a8),segFN=segments_1,generation=1}
   [junit4]   2> 2148612 T18283 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2148617 T18283 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2148618 T18283 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2148618 T18283 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2148619 T18283 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2148619 T18283 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2148619 T18283 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2148620 T18283 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2148621 T18283 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2148621 T18283 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2148622 T18283 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2148623 T18283 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2148623 T18283 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2148623 T18283 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2148624 T18283 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2148625 T18283 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2148626 T18283 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2148646 T18283 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2148652 T18283 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2148653 T18283 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2148654 T18283 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=36, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.20957049980281273]
   [junit4]   2> 2148654 T18283 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@121e14e9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@71cbd4a8),segFN=segments_1,generation=1}
   [junit4]   2> 2148655 T18283 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2148655 T18283 oass.SolrIndexSearcher.<init> Opening Searcher@25c80a4b[collection1] main
   [junit4]   2> 2148661 T18284 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@25c80a4b[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2148664 T18283 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2148665 T18218 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 2148666 T18218 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2148665 T18287 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:18179 collection:collection1 shard:shard1
   [junit4]   2> 2148668 T18287 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2148690 T18287 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2148693 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2148694 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2148694 T18287 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2148695 T18287 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2148694 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C3429 name=collection1 org.apache.solr.core.SolrCore@470cf912 url=https://127.0.0.1:18179/collection1 node=127.0.0.1:18179_ C3429_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:18179, core=collection1, node_name=127.0.0.1:18179_}
   [junit4]   2> 2148695 T18287 C3429 P18179 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:18179/collection1/
   [junit4]   2> 2148696 T18287 C3429 P18179 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2148696 T18287 C3429 P18179 oasc.SyncStrategy.syncToMe https://127.0.0.1:18179/collection1/ has no replicas
   [junit4]   2> 2148696 T18287 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:18179/collection1/ shard1
   [junit4]   2> 2148697 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2148697 T18287 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2148700 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2148701 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148701 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148701 T18282 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148701 T18264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148706 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2148709 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2148812 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148812 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148812 T18282 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148812 T18264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148858 T18287 oasc.ZkController.register We are https://127.0.0.1:18179/collection1/ and leader is https://127.0.0.1:18179/collection1/
   [junit4]   2> 2148858 T18287 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:18179
   [junit4]   2> 2148858 T18287 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2148859 T18287 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 2148859 T18287 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2148861 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2148861 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2148861 T18287 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2148861 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2148862 T18243 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2148863 T18243 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:18179",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18179_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 2148867 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2148974 T18251 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148974 T18264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148974 T18282 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2148974 T18242 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2149008 T18218 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 2149009 T18218 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2149016 T18218 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2149018 T18218 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:25285
   [junit4]   2> 2149022 T18218 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2149022 T18218 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2149023 T18218 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1393071048509
   [junit4]   2> 2149023 T18218 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1393071048509/'
   [junit4]   2> 2149073 T18218 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1393071048509/solr.xml
   [junit4]   2> 2149184 T18218 oasc.CoreContainer.<init> New CoreContainer 375581599
   [junit4]   2> 2149185 T18218 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1393071048509/]
   [junit4]   2> 2149187 T18218 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2149188 T18218 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2149188 T18218 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2149189 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2149190 T18218 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2149190 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2149191 T18218 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2149191 T18218 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2149192 T18218 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2149207 T18218 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2149208 T18218 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2149208 T18218 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2149209 T18218 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63284/solr
   [junit4]   2> 2149209 T18218 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2149210 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2149223 T18298 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76068ee0 name:ZooKeeperConnection Watcher:127.0.0.1:63284 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2149224 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2149256 T18218 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2149258 T18300 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@46bf28a6 name:ZooKeeperConnection Watcher:127.0.0.1:63284/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2149258 T18218 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2149272 T18218 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2150276 T18218 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25285_
   [junit4]   2> 2150279 T18218 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25285_
   [junit4]   2> 2150281 T18251 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2150282 T18300 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2150282 T18282 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2150282 T18242 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2150282 T18264 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2150300 T18301 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2150300 T18301 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2150302 T18242 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2150302 T18301 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 215

[...truncated too long message...]

ed
   [junit4]   2> 2232614 T18320 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty4;done=false>>]
   [junit4]   2> 2232615 T18320 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty4
   [junit4]   2> 2232615 T18320 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty4/index;done=false>>]
   [junit4]   2> 2232615 T18320 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.BasicDistributedZk2Test-1393071038016/jetty4/index
   [junit4]   2> 2232616 T18320 oasc.LeaderElector$1.process WARN  org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.<init>(DistributedQueue.java:63)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer.getInQueue(Overseer.java:1029)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:167)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:399)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:188)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:399)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:259)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1041)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:223)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:220)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:220)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:100)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.<init>(DistributedQueue.java:61)
   [junit4]   2> 	... 18 more
   [junit4]   2> 
   [junit4]   2> 2232617 T18320 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2232617 T18320 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2232617 T18320 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2232618 T18320 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Lucene3x, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=zh_SG, timezone=America/Mexico_City
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=1,free=229998960,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestDFRSimilarityFactory, RequiredFieldsTest, TestReloadAndDeleteDocs, TestAnalyzedSuggestions, DistributedSuggestComponentTest, PreAnalyzedUpdateProcessorTest, SolrCoreCheckLockOnStartupTest, TestManagedSchemaFieldResource, ShardSplitTest, HdfsBasicDistributedZk2Test, CurrencyFieldOpenExchangeTest, TestCodecSupport, TestSchemaNameResource, TestJmxMonitoredMap, TestCollationField, TestQuerySenderNoQuery, ConvertedLegacyTest, TestCoreDiscovery, OverseerCollectionProcessorTest, HdfsCollectionsAPIDistributedZkTest, TestBinaryField, WordBreakSolrSpellCheckerTest, TestDynamicFieldCollectionResource, UpdateParamsTest, ShardRoutingTest, OverseerTest, LeaderElectionTest, XmlUpdateRequestHandlerTest, AutoCommitTest, AddSchemaFieldsUpdateProcessorFactoryTest, NotRequiredUniqueKeyTest, TestSolrJ, TestSolrDeletionPolicy1, CoreAdminCreateDiscoverTest, SuggesterTest, StressHdfsTest, TestSchemaResource, TestPartialUpdateDeduplication, TestUtils, ShowFileRequestHandlerTest, TestSolr4Spatial, TestCloudManagedSchema, SOLR749Test, BJQParserTest, TestInfoStreamLogging, BasicDistributedZkTest, OverseerRolesTest, InfoHandlerTest, TestSolrXmlPersistence, TriLevelCompositeIdRoutingTest, OutputWriterTest, ResponseLogComponentTest, TestLMJelinekMercerSimilarityFactory, PeerSyncTest, TestSystemIdResolver, TestMultiCoreConfBootstrap, ChaosMonkeySafeLeaderTest, IndexBasedSpellCheckerTest, TestFieldTypeResource, TestDistributedSearch, DistributedQueryComponentCustomSortTest, DirectSolrSpellCheckerTest, TestDistributedMissingSort, ParsingFieldUpdateProcessorsTest, TestTrie, TestGroupingSearch, TestConfig, TestPerFieldSimilarity, TestQueryTypes, TestStressReorder, TestAtomicUpdateErrorCases, TestRandomFaceting, TestDefaultSimilarityFactory, SoftAutoCommitTest, CurrencyFieldXmlFileTest, TestFieldTypeCollectionResource, SpatialFilterTest, TestStressRecovery, TestDefaultSearchFieldResource, SpellPossibilityIteratorTest, TestLazyCores, ChangedSchemaMergeTest, FieldAnalysisRequestHandlerTest, SpellingQueryConverterTest, OpenCloseCoreStressTest, SchemaVersionSpecificBehaviorTest, TestSolrCoreProperties, DocumentAnalysisRequestHandlerTest, DistribCursorPagingTest, ChaosMonkeyNothingIsSafeTest, BasicDistributedZk2Test]
   [junit4] Completed on J1 in 95.16s, 1 test, 1 error <<< FAILURES!

[...truncated 563 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:499: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1278: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:911: There were test failures: 362 suites, 1563 tests, 1 error, 35 ignored (7 assumptions)

Total time: 74 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1907 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1907/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:29602/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:29602/collection1
	at __randomizedtesting.SeedInfo.seed([612B0DDEA9A60115:E0CD83C6DEF96129]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:503)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:168)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:146)
	at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:663)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:150)
	at java.net.SocketInputStream.read(SocketInputStream.java:121)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
	... 47 more




Build Log:
[...truncated 10511 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> 2472819 T23039 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 2472820 T23039 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 2472825 T23039 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solrtest-BasicDistributedZkTest-1392977423417
   [junit4]   2> 2472826 T23039 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 2472826 T23040 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2472927 T23039 oasc.ZkTestServer.run start zk server on port:29591
   [junit4]   2> 2472928 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2472932 T23046 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d4e404c name:ZooKeeperConnection Watcher:127.0.0.1:29591 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2472932 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2472932 T23039 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2472951 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2472953 T23048 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f54122c name:ZooKeeperConnection Watcher:127.0.0.1:29591/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2472954 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2472954 T23039 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2472957 T23039 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2472968 T23039 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2472969 T23039 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2472971 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2472972 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2472982 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2472983 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2473086 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2473087 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2473089 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2473089 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2473099 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2473099 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2473101 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2473102 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2473104 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2473104 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 2473106 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2473107 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2473114 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2473114 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2473116 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2473117 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2473119 T23039 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2473119 T23039 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2473123 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2473125 T23050 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ff375f9 name:ZooKeeperConnection Watcher:127.0.0.1:29591/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2473125 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2473328 T23039 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2473333 T23039 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2473340 T23039 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:29595
   [junit4]   2> 2473342 T23039 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2473343 T23039 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2473343 T23039 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392977423728
   [junit4]   2> 2473344 T23039 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392977423728/'
   [junit4]   2> 2473374 T23039 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392977423728/solr.xml
   [junit4]   2> 2473442 T23039 oasc.CoreContainer.<init> New CoreContainer 2054123061
   [junit4]   2> 2473443 T23039 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392977423728/]
   [junit4]   2> 2473445 T23039 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2473445 T23039 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2473445 T23039 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2473446 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2473446 T23039 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2473447 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2473447 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2473447 T23039 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2473448 T23039 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2473458 T23039 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2473458 T23039 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2473459 T23039 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2473459 T23039 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:29591/solr
   [junit4]   2> 2473459 T23039 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2473460 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2473463 T23061 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@41bd9aa6 name:ZooKeeperConnection Watcher:127.0.0.1:29591 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2473463 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2473473 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2473475 T23063 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2bae4cf4 name:ZooKeeperConnection Watcher:127.0.0.1:29591/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2473475 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2473477 T23039 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2473487 T23039 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2473490 T23039 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2473498 T23039 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:29595_
   [junit4]   2> 2473499 T23039 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:29595_
   [junit4]   2> 2473501 T23039 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 2473503 T23039 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2473514 T23039 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:29595_
   [junit4]   2> 2473514 T23039 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2473516 T23039 oasc.Overseer.start Overseer (id=91290168421449732-127.0.0.1:29595_-n_0000000000) starting
   [junit4]   2> 2473519 T23039 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2473533 T23065 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2473534 T23039 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2473535 T23065 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2473536 T23039 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2473538 T23039 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2473540 T23064 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2473586 T23066 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2473586 T23066 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2473588 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2473588 T23066 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2473589 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2473590 T23064 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:29595",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:29595_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2473590 T23064 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 2473591 T23064 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2473599 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2473600 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2474590 T23066 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2474590 T23066 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392977423728/collection1
   [junit4]   2> 2474590 T23066 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 2474591 T23066 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2474591 T23066 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2474592 T23066 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2474592 T23066 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392977423728/collection1/'
   [junit4]   2> 2474593 T23066 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392977423728/collection1/lib/README' to classloader
   [junit4]   2> 2474594 T23066 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392977423728/collection1/lib/classes/' to classloader
   [junit4]   2> 2474642 T23066 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 2474694 T23066 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2474800 T23066 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2474809 T23066 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2475275 T23066 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2475276 T23066 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2475277 T23066 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2475286 T23066 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2475289 T23066 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2475298 T23066 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2475302 T23066 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2475306 T23066 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2475306 T23066 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2475307 T23066 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2475307 T23066 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2475307 T23066 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2475308 T23066 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2475308 T23066 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2475308 T23066 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392977423728/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/control/data/
   [junit4]   2> 2475309 T23066 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@609b49b0
   [junit4]   2> 2475309 T23066 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/control/data
   [junit4]   2> 2475310 T23066 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/control/data/index/
   [junit4]   2> 2475310 T23066 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2475310 T23066 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/control/data/index
   [junit4]   2> 2475311 T23066 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@66c649dd
   [junit4]   2> 2475312 T23066 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5fd36968 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@641b80e1),segFN=segments_1,generation=1}
   [junit4]   2> 2475312 T23066 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2475315 T23066 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2475316 T23066 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2475316 T23066 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2475316 T23066 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2475317 T23066 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2475317 T23066 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2475317 T23066 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2475318 T23066 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2475318 T23066 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2475319 T23066 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2475319 T23066 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2475319 T23066 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2475319 T23066 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2475320 T23066 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2475320 T23066 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2475321 T23066 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2475334 T23066 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2475338 T23066 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2475339 T23066 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2475339 T23066 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=12, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 2475340 T23066 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5fd36968 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@641b80e1),segFN=segments_1,generation=1}
   [junit4]   2> 2475340 T23066 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2475341 T23066 oass.SolrIndexSearcher.<init> Opening Searcher@7fcdf8f9[collection1] main
   [junit4]   2> 2475343 T23067 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7fcdf8f9[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2475344 T23066 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2475345 T23039 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 2475346 T23039 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2475345 T23070 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:29595 collection:control_collection shard:shard1
   [junit4]   2> 2475347 T23070 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 2475350 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2475369 T23072 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6527721e name:ZooKeeperConnection Watcher:127.0.0.1:29591/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2475369 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2475371 T23039 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2475371 T23070 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2475373 T23039 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2475374 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2475374 T23070 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2475374 T23070 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2884 name=collection1 org.apache.solr.core.SolrCore@5e15091c url=https://127.0.0.1:29595/collection1 node=127.0.0.1:29595_ C2884_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:29595, core=collection1, node_name=127.0.0.1:29595_}
   [junit4]   2> 2475374 T23070 C2884 P29595 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:29595/collection1/
   [junit4]   2> 2475374 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2475374 T23070 C2884 P29595 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2475375 T23070 C2884 P29595 oasc.SyncStrategy.syncToMe https://127.0.0.1:29595/collection1/ has no replicas
   [junit4]   2> 2475375 T23070 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:29595/collection1/ shard1
   [junit4]   2> 2475375 T23070 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 2475378 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2475389 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2475389 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2475400 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2475410 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2475513 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2475514 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2475552 T23070 oasc.ZkController.register We are https://127.0.0.1:29595/collection1/ and leader is https://127.0.0.1:29595/collection1/
   [junit4]   2> 2475552 T23070 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:29595
   [junit4]   2> 2475552 T23070 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2475552 T23070 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 2475553 T23070 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2475554 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2475554 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2475555 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2475555 T23070 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2475556 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2475557 T23064 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:29595",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:29595_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 2475559 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2475571 T23039 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 2475573 T23039 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2475577 T23039 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2475578 T23039 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:29599
   [junit4]   2> 2475580 T23039 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2475581 T23039 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2475581 T23039 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392977425965
   [junit4]   2> 2475582 T23039 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392977425965/'
   [junit4]   2> 2475616 T23039 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392977425965/solr.xml
   [junit4]   2> 2475662 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2475662 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2475684 T23039 oasc.CoreContainer.<init> New CoreContainer 1819362832
   [junit4]   2> 2475685 T23039 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392977425965/]
   [junit4]   2> 2475687 T23039 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2475687 T23039 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2475687 T23039 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2475688 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2475688 T23039 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2475688 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2475689 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2475689 T23039 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2475689 T23039 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2475700 T23039 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2475700 T23039 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2475701 T23039 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2475701 T23039 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:29591/solr
   [junit4]   2> 2475702 T23039 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2475702 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2475704 T23083 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d71502d name:ZooKeeperConnection Watcher:127.0.0.1:29591 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2475705 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2475707 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2475715 T23085 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@32cc18c9 name:ZooKeeperConnection Watcher:127.0.0.1:29591/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2475715 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2475727 T23039 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2476730 T23039 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:29599_
   [junit4]   2> 2476734 T23039 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:29599_
   [junit4]   2> 2476738 T23072 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2476738 T23085 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2476738 T23063 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2476748 T23086 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2476749 T23086 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2476750 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2476750 T23086 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2476751 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2476751 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2476752 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2476753 T23064 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:29599",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:29599_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2476753 T23064 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 2476754 T23064 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 2476757 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2476859 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2476859 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2476859 T23085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2477752 T23086 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2477752 T23086 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392977425965/collection1
   [junit4]   2> 2477752 T23086 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2477753 T23086 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2477753 T23086 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2477754 T23086 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2477754 T23086 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392977425965/collection1/'
   [junit4]   2> 2477756 T23086 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392977425965/collection1/lib/classes/' to classloader
   [junit4]   2> 2477756 T23086 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392977425965/collection1/lib/README' to classloader
   [junit4]   2> 2477804 T23086 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 2477854 T23086 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2477955 T23086 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2477964 T23086 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2478443 T23086 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2478444 T23086 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2478445 T23086 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2478454 T23086 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2478457 T23086 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2478467 T23086 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2478472 T23086 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2478478 T23086 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2478479 T23086 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2478479 T23086 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2478480 T23086 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2478480 T23086 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2478480 T23086 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2478481 T23086 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2478481 T23086 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392977425965/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty1/
   [junit4]   2> 2478481 T23086 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@609b49b0
   [junit4]   2> 2478482 T23086 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty1
   [junit4]   2> 2478482 T23086 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty1/index/
   [junit4]   2> 2478483 T23086 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 2478483 T23086 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty1/index
   [junit4]   2> 2478483 T23086 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@136ddc4c
   [junit4]   2> 2478484 T23086 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@458492e1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@119a0b8b),segFN=segments_1,generation=1}
   [junit4]   2> 2478484 T23086 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2478488 T23086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2478489 T23086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2478489 T23086 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2478489 T23086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2478490 T23086 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2478490 T23086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2478490 T23086 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2478491 T23086 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2478491 T23086 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2478491 T23086 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2478492 T23086 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2478492 T23086 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2478492 T23086 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2478493 T23086 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2478493 T23086 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2478494 T23086 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2478505 T23086 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2478509 T23086 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2478509 T23086 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2478510 T23086 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=12, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 2478510 T23086 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@458492e1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@119a0b8b),segFN=segments_1,generation=1}
   [junit4]   2> 2478511 T23086 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2478511 T23086 oass.SolrIndexSearcher.<init> Opening Searcher@251fbf74[collection1] main
   [junit4]   2> 2478514 T23087 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@251fbf74[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2478516 T23086 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2478517 T23039 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 2478517 T23039 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2478517 T23090 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:29599 collection:collection1 shard:shard2
   [junit4]   2> 2478519 T23090 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 2478540 T23090 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 2478551 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2478551 T23090 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2478552 T23090 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2478551 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C2885 name=collection1 org.apache.solr.core.SolrCore@37a0982e url=https://127.0.0.1:29599/collection1 node=127.0.0.1:29599_ C2885_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:29599, core=collection1, node_name=127.0.0.1:29599_}
   [junit4]   2> 2478552 T23090 C2885 P29599 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:29599/collection1/
   [junit4]   2> 2478552 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2478552 T23090 C2885 P29599 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2478553 T23090 C2885 P29599 oasc.SyncStrategy.syncToMe https://127.0.0.1:29599/collection1/ has no replicas
   [junit4]   2> 2478553 T23090 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:29599/collection1/ shard2
   [junit4]   2> 2478553 T23090 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 2478553 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2478562 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2478563 T23085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2478563 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2478563 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2478566 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2478569 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2478672 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2478672 T23085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2478672 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2478711 T23039 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 2478712 T23039 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2478716 T23039 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2478718 T23039 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:29602
   [junit4]   2> 2478719 T23090 oasc.ZkController.register We are https://127.0.0.1:29599/collection1/ and leader is https://127.0.0.1:29599/collection1/
   [junit4]   2> 2478720 T23090 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:29599
   [junit4]   2> 2478720 T23090 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2478720 T23090 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 2478720 T23090 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2478721 T23039 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2478721 T23039 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2478722 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2478722 T23039 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392977429111
   [junit4]   2> 2478722 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2478722 T23090 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2478722 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2478722 T23039 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392977429111/'
   [junit4]   2> 2478723 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2478724 T23064 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:29599",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:29599_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 2478726 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2478754 T23039 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392977429111/solr.xml
   [junit4]   2> 2478821 T23039 oasc.CoreContainer.<init> New CoreContainer 203812585
   [junit4]   2> 2478822 T23039 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392977429111/]
   [junit4]   2> 2478823 T23039 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2478824 T23039 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2478824 T23039 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2478824 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2478825 T23039 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2478825 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2478825 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2478826 T23039 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2478826 T23039 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2478828 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2478828 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2478828 T23085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2478836 T23039 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2478837 T23039 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2478837 T23039 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2478838 T23039 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:29591/solr
   [junit4]   2> 2478838 T23039 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2478839 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2478841 T23101 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6ba620d4 name:ZooKeeperConnection Watcher:127.0.0.1:29591 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2478841 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2478843 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2478851 T23103 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b44405 name:ZooKeeperConnection Watcher:127.0.0.1:29591/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2478852 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2478860 T23039 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2479862 T23039 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:29602_
   [junit4]   2> 2479864 T23039 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:29602_
   [junit4]   2> 2479868 T23072 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2479868 T23085 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2479868 T23063 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2479868 T23103 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2479886 T23104 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 2479887 T23104 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2479888 T23104 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2479888 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2479888 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2479888 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2479889 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2479890 T23064 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:29602",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:29602_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2479890 T23064 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 2479890 T23064 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2479893 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2479996 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2479996 T23103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2479996 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2479997 T23085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2480889 T23104 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2480889 T23104 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392977429111/collection1
   [junit4]   2> 2480890 T23104 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2480890 T23104 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2480891 T23104 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2480892 T23104 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2480892 T23104 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392977429111/collection1/'
   [junit4]   2> 2480893 T23104 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392977429111/collection1/lib/README' to classloader
   [junit4]   2> 2480894 T23104 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392977429111/collection1/lib/classes/' to classloader
   [junit4]   2> 2480961 T23104 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 2481030 T23104 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2481131 T23104 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2481143 T23104 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2481830 T23104 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2481831 T23104 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2481832 T23104 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2481845 T23104 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2481850 T23104 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2481863 T23104 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2481868 T23104 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2481872 T23104 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2481873 T23104 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2481874 T23104 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2481874 T23104 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2481875 T23104 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2481875 T23104 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2481876 T23104 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2481876 T23104 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392977429111/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty2/
   [junit4]   2> 2481876 T23104 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@609b49b0
   [junit4]   2> 2481879 T23104 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty2
   [junit4]   2> 2481879 T23104 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty2/index/
   [junit4]   2> 2481880 T23104 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 2481880 T23104 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty2/index
   [junit4]   2> 2481881 T23104 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@425ec172
   [junit4]   2> 2481882 T23104 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5698e999 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5aee4568),segFN=segments_1,generation=1}
   [junit4]   2> 2481882 T23104 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2481888 T23104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2481888 T23104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2481889 T23104 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2481889 T23104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2481889 T23104 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2481890 T23104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2481890 T23104 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2481891 T23104 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2481891 T23104 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2481892 T23104 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2481893 T23104 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2481893 T23104 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2481894 T23104 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2481895 T23104 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2481895 T23104 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2481896 T23104 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2481913 T23104 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2481919 T23104 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2481919 T23104 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2481920 T23104 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=12, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 2481921 T23104 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5698e999 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5aee4568),segFN=segments_1,generation=1}
   [junit4]   2> 2481922 T23104 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2481922 T23104 oass.SolrIndexSearcher.<init> Opening Searcher@5c853739[collection1] main
   [junit4]   2> 2481927 T23105 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5c853739[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2481930 T23104 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2481931 T23039 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 2481932 T23039 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2481931 T23108 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:29602 collection:collection1 shard:shard1
   [junit4]   2> 2481933 T23108 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2481956 T23108 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2481967 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2481968 T23108 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2481968 T23108 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2481968 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C2886 name=collection1 org.apache.solr.core.SolrCore@285ef6aa url=https://127.0.0.1:29602/collection1 node=127.0.0.1:29602_ C2886_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:29602, core=collection1, node_name=127.0.0.1:29602_}
   [junit4]   2> 2481968 T23108 C2886 P29602 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:29602/collection1/
   [junit4]   2> 2481969 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2481969 T23108 C2886 P29602 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2481970 T23108 C2886 P29602 oasc.SyncStrategy.syncToMe https://127.0.0.1:29602/collection1/ has no replicas
   [junit4]   2> 2481970 T23108 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:29602/collection1/ shard1
   [junit4]   2> 2481970 T23108 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2481970 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2481980 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2481981 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2481981 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2481981 T23085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2481981 T23103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2481984 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2481987 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2482090 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2482090 T23103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2482090 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2482091 T23085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2482139 T23108 oasc.ZkController.register We are https://127.0.0.1:29602/collection1/ and leader is https://127.0.0.1:29602/collection1/
   [junit4]   2> 2482139 T23108 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:29602
   [junit4]   2> 2482139 T23108 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2482139 T23108 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 2482140 T23108 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2482141 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2482141 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2482142 T23108 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2482142 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2482143 T23064 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2482144 T23064 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:29602",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:29602_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 2482152 T23063 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2482156 T23039 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 2482157 T23039 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2482161 T23039 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 2482162 T23039 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:29605
   [junit4]   2> 2482164 T23039 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2482164 T23039 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2482165 T23039 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392977432527
   [junit4]   2> 2482165 T23039 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392977432527/'
   [junit4]   2> 2482197 T23039 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392977432527/solr.xml
   [junit4]   2> 2482255 T23072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2482255 T23085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2482255 T23063 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2482255 T23103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2482264 T23039 oasc.CoreContainer.<init> New CoreContainer 846345369
   [junit4]   2> 2482265 T23039 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392977432527/]
   [junit4]   2> 2482267 T23039 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2482267 T23039 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2482267 T23039 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2482268 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2482268 T23039 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2482268 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2482269 T23039 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2482269 T23039 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2482270 T23039 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2482280 T23039 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2482280 T23039 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2482281 T23039 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2482281 T23039 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:29591/solr
   [junit4]   2> 2482281 T23039 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2482282 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2482284 T23119 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11c158ae name:ZooKeeperConnection Watcher:127.0.0.1:29591 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2482285 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2482286 T23039 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2482288 T23121 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3da7c784 name:ZooKeeperConnection Watcher:127.0.0.1:29591/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2482288 T23039 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2482299 T23039 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2483302 T23039 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:29605_
   [junit4]   2> 2483333 T23039 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:29605_
   [junit4]   2> 2483338 T23072 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2483338 T23103 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2483338 T23085 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2483338 T23121 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2483338 T23063 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2483353 T23122

[...truncated too long message...]

oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=50,autocommits=0,soft autocommits=1,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=76,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=10}
   [junit4]   2> 2604363 T23141 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2604364 T23141 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2604364 T23141 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2604365 T23141 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2604365 T23141 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2604366 T23141 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty4/index;done=false>>]
   [junit4]   2> 2604366 T23141 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty4/index
   [junit4]   2> 2604366 T23141 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty4;done=false>>]
   [junit4]   2> 2604367 T23141 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.BasicDistributedZkTest-1392977423417/jetty4
   [junit4]   2> 2604367 T23141 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection2/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2604367 T23141 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection3/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2604368 T23141 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2604369 T23141 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=zh_TW, timezone=Asia/Pontianak
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=1,free=170903896,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestCopyFieldCollectionResource, TestFastOutputStream, DistributedQueryComponentCustomSortTest, RequiredFieldsTest, SolrRequestParserTest, AliasIntegrationTest, TestValueSourceCache, DistributedDebugComponentTest, BlockCacheTest, AlternateDirectoryTest, DateFieldTest, SimpleFacetsTest, SignatureUpdateProcessorFactoryTest, TestRangeQuery, StatsComponentTest, TestCSVResponseWriter, MigrateRouteKeyTest, PreAnalyzedFieldTest, InfoHandlerTest, TestImplicitCoreProperties, TestDistributedMissingSort, ClusterStateUpdateTest, RemoteQueryErrorTest, TestQuerySenderListener, TestMergePolicyConfig, TestJoin, SuggesterFSTTest, PolyFieldTest, DistributedSuggestComponentTest, TestOmitPositions, TestSolrXMLSerializer, TestCustomSort, TestQuerySenderNoQuery, TestLMJelinekMercerSimilarityFactory, NotRequiredUniqueKeyTest, SortByFunctionTest, ZkControllerTest, TestSolrQueryParserDefaultOperatorResource, TestBM25SimilarityFactory, ShardSplitTest, TestIndexingPerformance, HdfsUnloadDistributedZkTest, TestRandomFaceting, TestCharFilters, LegacyHTMLStripCharFilterTest, ZkNodePropsTest, TestSearchPerf, TestFreeTextSuggestions, TestCollapseQParserPlugin, DeleteReplicaTest, RAMDirectoryFactoryTest, TriLevelCompositeIdRoutingTest, TestFoldingMultitermQuery, TestReplicationHandler, ZkCLITest, ShardRoutingTest, CoreContainerCoreInitFailuresTest, AssignTest, OpenCloseCoreStressTest, TestCollationField, AnalysisAfterCoreReloadTest, BasicDistributedZk2Test, TestClassNameShortening, JSONWriterTest, FieldAnalysisRequestHandlerTest, DirectSolrSpellCheckerTest, TestWordDelimiterFilterFactory, SpellingQueryConverterTest, DocValuesMultiTest, TestMaxScoreQueryParser, TestManagedSchema, DOMUtilTest, AnalysisErrorHandlingTest, EnumFieldTest, DocValuesTest, TestDistributedGrouping, TestComponentsName, BadCopyFieldTest, TestJmxIntegration, OutputWriterTest, TestElisionMultitermQuery, TestSolrDeletionPolicy2, IndexSchemaRuntimeFieldTest, ReturnFieldsTest, TestFastWriter, TestShardHandlerFactory, CachingDirectoryFactoryTest, MinimalSchemaTest, DistribCursorPagingTest, ChaosMonkeyNothingIsSafeTest, CollectionsAPIDistributedZkTest, BasicDistributedZkTest]
   [junit4] Completed on J1 in 131.98s, 1 test, 1 error <<< FAILURES!

[...truncated 554 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1272: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:905: There were test failures: 362 suites, 1563 tests, 1 error, 35 ignored (7 assumptions)

Total time: 81 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1906 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1906/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.OverseerTest.testOverseerFailure

Error Message:
KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1

Stack Trace:
org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
	at __randomizedtesting.SeedInfo.seed([8486475CAB1AA0D4:808EC8AFB9BF4FF5]:0)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
	at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:429)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:426)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:383)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:370)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)




Build Log:
[...truncated 10571 lines...]
   [junit4] Suite: org.apache.solr.cloud.OverseerTest
   [junit4]   2> 2573481 T27650 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 2573482 T27650 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solrtest-OverseerTest-1392901646955
   [junit4]   2> 2573483 T27650 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 2573489 T27650 oas.SolrTestCaseJ4.setUp ###Starting testStateChange
   [junit4]   2> 2573490 T27650 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 2573491 T27651 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2573593 T27650 oasc.ZkTestServer.run start zk server on port:48392
   [junit4]   2> 2573594 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2573599 T27657 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f9c0e77 name:ZooKeeperConnection Watcher:127.0.0.1:48392/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2573600 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2573601 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2573603 T27659 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e64cca name:ZooKeeperConnection Watcher:127.0.0.1:48392 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2573604 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2573657 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2573660 T27661 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fc3531c name:ZooKeeperConnection Watcher:127.0.0.1:48392 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2573660 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2573661 T27650 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2573679 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2573683 T27650 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2573691 T27650 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2573694 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2573697 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2573700 T27663 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e3d533 name:ZooKeeperConnection Watcher:127.0.0.1:48392/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2573700 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2573702 T27650 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 2573706 T27650 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2573802 T27650 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48392_solr
   [junit4]   2> 2573803 T27650 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2573806 T27650 oasc.Overseer.start Overseer (id=91285202336088067-127.0.0.1:48392_solr-n_0000000000) starting
   [junit4]   2> 2573838 T27650 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2573844 T27650 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2573860 T27650 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2573864 T27665 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2573865 T27664 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2573865 T27665 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2573874 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2573875 T27664 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2573876 T27664 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "roles":"",
   [junit4]   2> 	  "state":"recovering"}
   [junit4]   2> 2573876 T27664 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2573880 T27663 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2573881 T27657 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 2573927 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2573930 T27663 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2573932 T27664 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2573933 T27664 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "roles":"",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 2573933 T27664 oasc.Overseer$ClusterStateUpdater.updateState node=core_node1 is already registered
   [junit4]   2> 2573933 T27664 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2573936 T27663 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2574041 T27657 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 2574086 T27652 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1444f686a7e0003, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 2574088 T27650 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:48392 48392
   [junit4]   2> 2574252 T27650 oas.SolrTestCaseJ4.tearDown ###Ending testStateChange
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2574261 T27650 oas.SolrTestCaseJ4.setUp ###Starting testShardLeaderChange
   [junit4]   2> 2574262 T27650 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 2574263 T27666 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2574364 T27650 oasc.ZkTestServer.run start zk server on port:34555
   [junit4]   2> 2574365 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2574399 T27672 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31252dfe name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2574400 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2574401 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2574404 T27674 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@138e5772 name:ZooKeeperConnection Watcher:127.0.0.1:34555 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2574404 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2574449 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2574454 T27676 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ca7ef03 name:ZooKeeperConnection Watcher:127.0.0.1:34555 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2574454 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2574455 T27650 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2574458 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2574463 T27677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2574464 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2574465 T27679 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19dcc950 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2574465 T27677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2574466 T27681 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f231860 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2574466 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2574466 T27677 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 2574467 T27650 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2574469 T27677 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2574470 T27650 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2574473 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2574474 T27677 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34555_solr
   [junit4]   2> 2574474 T27677 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2574476 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 2574477 T27677 oasc.Overseer.start Overseer (id=91285202386419715-127.0.0.1:34555_solr-n_0000000000) starting
   [junit4]   2> 2574479 T27681 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2574480 T27677 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2574481 T27650 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2574483 T27677 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2574487 T27677 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2574490 T27683 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2574490 T27677 oasc.OverseerTest$OverseerRestarter.run Killing overseer.
   [junit4]   2> 2574491 T27683 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2574491 T27682 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2574492 T27677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2574494 T27685 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a2e9735 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2574494 T27677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2574498 T27677 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34555_solr
   [junit4]   2> 2574498 T27677 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2574500 T27677 oasc.Overseer.start Overseer (id=91285202386419717-127.0.0.1:34555_solr-n_0000000001) starting
   [junit4]   2> 2574505 T27687 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2574506 T27686 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2574507 T27687 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2574510 T27686 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2574510 T27686 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2574511 T27686 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 2574511 T27686 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2574514 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2574515 T27681 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2574986 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2575015 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2575031 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575036 T27686 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2575037 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575037 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2575047 T27686 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2575048 T27686 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2575060 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2575060 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575062 T27689 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f3e08dc name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2575062 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2575064 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2575066 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 2575071 T27681 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2575071 T27689 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2575073 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575073 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2575075 T27686 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2575075 T27686 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2575085 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575086 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2575188 T27681 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2575188 T27689 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2575289 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2575294 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575295 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575295 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575296 T27686 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2575296 T27686 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node2",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node2/solr/"}
   [junit4]   2> 2575297 T27686 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 2575297 T27686 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2575306 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575408 T27681 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2575408 T27689 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2575795 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2575811 T27689 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2575811 T27681 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2575812 T27681 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2575812 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2575813 T27689 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2575813 T27681 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2575825 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575826 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575826 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575827 T27686 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2575830 T27685 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2575932 T27689 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2575992 T27683 oasc.OverseerCollectionProcessor.run ERROR Unable to prioritize overseer  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /roles.json
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1041)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:223)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:220)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:220)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionProcessor.prioritizeOverseerNodes(OverseerCollectionProcessor.java:227)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:178)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 2575993 T27682 oasc.Overseer$ClusterStateUpdater.amILeader  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1151)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:271)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:271)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:398)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:153)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 2575993 T27683 oasc.OverseerCollectionProcessor.amILeader  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1151)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:271)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:271)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionProcessor.amILeader(OverseerCollectionProcessor.java:357)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionProcessor.run(OverseerCollectionProcessor.java:185)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 2575994 T27682 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=91285202386419715-127.0.0.1:34555_solr-n_0000000000) am no longer a leader.
   [junit4]   2> 2575994 T27683 oasc.OverseerCollectionProcessor.amILeader According to ZK I (id=91285202386419715-127.0.0.1:34555_solr-n_0000000000) am no longer a leader.
   [junit4]   2> 2576015 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576018 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2576020 T27691 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38797a21 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2576020 T27677 oasc.OverseerTest$OverseerRestarter.run Killing overseer.
   [junit4]   2> 2576020 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2576022 T27677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2576022 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2576030 T27693 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49ae0189 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2576031 T27677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2576032 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 2576035 T27677 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34555_solr
   [junit4]   2> 2576036 T27677 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2576045 T27689 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2576045 T27691 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2576046 T27677 oasc.Overseer.start Overseer (id=91285202386419720-127.0.0.1:34555_solr-n_0000000002) starting
   [junit4]   2> 2576057 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2576061 T27695 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2576062 T27694 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2576063 T27695 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2576064 T27694 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576065 T27694 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2576065 T27691 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2576065 T27689 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2576065 T27694 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2576067 T27691 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2576068 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576068 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2576073 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576073 T27691 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2576074 T27694 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576075 T27694 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2576075 T27694 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2576075 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2576077 T27697 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44920651 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2576078 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2576081 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2576082 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576084 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 2576087 T27697 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2576087 T27691 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2576090 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576090 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2576091 T27694 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2576091 T27694 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2576094 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576096 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576198 T27697 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2576198 T27691 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2576298 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576303 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576303 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2576303 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576304 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576304 T27694 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576305 T27694 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node2",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node2/solr/"}
   [junit4]   2> 2576306 T27694 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2576323 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576324 T27697 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2576324 T27691 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2576325 T27691 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2576325 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576326 T27697 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2576326 T27691 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2576332 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576335 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576438 T27697 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2576528 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576531 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2576533 T27699 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@403e35c1 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2576533 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2576535 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2576537 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 2576542 T27699 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2576542 T27697 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2576551 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576551 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2576552 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576553 T27694 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576553 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576554 T27694 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2576554 T27694 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2576566 T27677 oasc.OverseerTest$OverseerRestarter.run Killing overseer.
   [junit4]   2> 2576567 T27693 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576568 T27677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2576570 T27701 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1694aa72 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2576570 T27699 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2576570 T27697 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2576570 T27677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2576572 T27699 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2576572 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2576575 T27677 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34555_solr
   [junit4]   2> 2576575 T27677 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2576577 T27677 oasc.Overseer.start Overseer (id=91285202386419723-127.0.0.1:34555_solr-n_0000000003) starting
   [junit4]   2> 2576581 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2576582 T27703 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@59c052ec name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2576583 T27705 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2576583 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2576584 T27705 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2576584 T27704 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576585 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2576585 T27704 oasc.Overseer$ClusterStateUpdater.run Replaying operations from work queue.
   [junit4]   2> 2576586 T27704 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2576587 T27704 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2576588 T27699 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2576588 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 2576588 T27703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2576590 T27704 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2576592 T27699 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2576592 T27704 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576592 T27703 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2576593 T27704 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2576594 T27704 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2576594 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2576594 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576597 T27703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2576597 T27699 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2576601 T27704 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576601 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576604 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576605 T27704 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2576605 T27704 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2576608 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576711 T27703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2576711 T27699 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2576804 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576816 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576816 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2576817 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576818 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576821 T27704 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576823 T27704 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node2",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node2/solr/"}
   [junit4]   2> 2576823 T27704 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2576836 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576837 T27703 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2576837 T27699 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2576848 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2576849 T27703 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2576854 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576858 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2576960 T27703 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2577051 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577053 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2577056 T27707 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@42a15ada name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2577056 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2577058 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2577060 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 2577065 T27707 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2577065 T27703 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2577067 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577067 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2577067 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577069 T27704 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577069 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577070 T27704 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577070 T27704 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577086 T27701 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577088 T27703 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2577088 T27707 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2577089 T27677 oasc.OverseerTest$OverseerRestarter.run Killing overseer.
   [junit4]   2> 2577090 T27677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2577090 T27707 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2577091 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2577092 T27709 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@193a2062 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2577093 T27677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2577096 T27677 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34555_solr
   [junit4]   2> 2577097 T27677 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2577109 T27677 oasc.Overseer.start Overseer (id=91285202386419726-127.0.0.1:34555_solr-n_0000000004) starting
   [junit4]   2> 2577110 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2577117 T27711 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@517312 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2577118 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2577128 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2577130 T27713 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2577131 T27713 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2577131 T27712 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577131 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 2577132 T27712 oasc.Overseer$ClusterStateUpdater.run Replaying operations from work queue.
   [junit4]   2> 2577133 T27712 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577133 T27712 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577135 T27707 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2577136 T27711 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2577136 T27707 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577136 T27711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577146 T27712 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2577147 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2577148 T27712 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577149 T27712 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577150 T27712 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577161 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577162 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577162 T27707 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577162 T27711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577166 T27712 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577169 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577170 T27712 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577170 T27712 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577173 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577187 T27704 oasc.Overseer$ClusterStateUpdater.run WARN Solr cannot talk to ZK, exiting Overseer main queue loop org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.orderedChildren(DistributedQueue.java:89)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:411)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:195)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 2577275 T27711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577276 T27707 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577364 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577368 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577368 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2577369 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577370 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577371 T27712 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577372 T27712 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node2",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node2/solr/"}
   [junit4]   2> 2577373 T27712 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577376 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577377 T27707 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2577377 T27711 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2577378 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577379 T27711 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2577384 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577388 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577490 T27711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2577580 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577583 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2577585 T27715 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b411946 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2577585 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2577587 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2577589 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 2577593 T27711 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2577593 T27715 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2577595 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577595 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2577596 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577597 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577597 T27712 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577598 T27712 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577598 T27712 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577601 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577603 T27715 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2577603 T27711 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2577605 T27715 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2577605 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577605 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2577607 T27712 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577607 T27712 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577610 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577611 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577614 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2577614 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577616 T27717 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@139331c0 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2577616 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2577617 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2577620 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 2577623 T27715 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2577623 T27717 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2577626 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577626 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2577627 T27712 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577628 T27712 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577631 T27709 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577632 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577636 T27677 oasc.OverseerTest$OverseerRestarter.run Killing overseer.
   [junit4]   2> 2577637 T27677 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2577639 T27719 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@370c28ce name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2577640 T27677 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2577643 T27677 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34555_solr
   [junit4]   2> 2577643 T27677 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2577645 T27677 oasc.Overseer.start Overseer (id=91285202386419730-127.0.0.1:34555_solr-n_0000000005) starting
   [junit4]   2> 2577650 T27721 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2577650 T27721 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 2577651 T27720 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577652 T27720 oasc.Overseer$ClusterStateUpdater.run Replaying operations from work queue.
   [junit4]   2> 2577653 T27720 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577653 T27720 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577654 T27715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577654 T27717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577657 T27720 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577657 T27720 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577658 T27717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577658 T27715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577661 T27717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577661 T27715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577664 T27720 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state1",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 2577664 T27720 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577665 T27717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577665 T27715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577667 T27720 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2577732 T27712 oasc.Overseer$ClusterStateUpdater.run WARN Solr cannot talk to ZK, exiting Overseer main queue loop org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.orderedChildren(DistributedQueue.java:89)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:411)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:195)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 2577834 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577840 T27719 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577840 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2577842 T27720 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577843 T27720 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"state2",
   [junit4]   2> 	  "node_name":"node2",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node2/solr/"}
   [junit4]   2> 2577843 T27720 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 2577855 T27719 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577864 T27715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577864 T27717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2577866 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577866 T27717 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 2577875 T27717 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2577880 T27719 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577882 T27720 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2577885 T27719 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2577988 T27717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2578068 T27694 oasc.Overseer$ClusterStateUpdater.run WARN Solr cannot talk to ZK, exiting Overseer main queue loop org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.orderedChildren(DistributedQueue.java:89)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:411)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:195)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 2578076 T27650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2578078 T27650 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2578080 T27723 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a4c9465 name:ZooKeeperConnection Watcher:127.0.0.1:34555/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2578081 T27650 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2578082 T27650 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2578085 T27650 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 2578088 T27717 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2578088 T27723 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2578096 T27719 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2578096 T27650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2578096 T27719 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2578097 T27719 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeCh

[...truncated too long message...]

on1",
   [junit4]   2> 	  "core":"core2",
   [junit4]   2> 	  "roles":"",
   [junit4]   2> 	  "state":"recovering"}
   [junit4]   2> 2597773 T28132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 2597774 T28139 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2597775 T28139 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2597776 T28139 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "shard":"s1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"core3",
   [junit4]   2> 	  "roles":"",
   [junit4]   2> 	  "state":"recovering"}
   [junit4]   2> 2597778 T28138 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2597779 T28132 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 2597872 T27650 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26932 26932
   [junit4]   2> 2597882 T28108 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2598047 T27650 oas.SolrTestCaseJ4.tearDown ###Ending testReplay
   [junit4]   2> 2598047 T28108 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2598049 T28108 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2598050 T27650 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 2598050 T27650 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 2601053 T27650 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=pt_BR, timezone=America/Paramaribo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=1,free=255331328,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [SolrTestCaseJ4Test, EchoParamsTest, TestUpdate, QueryElevationComponentTest, TestCollationField, NumericFieldsTest, ShowFileRequestHandlerTest, OpenCloseCoreStressTest, TestFieldResource, SpellCheckCollatorTest, LeaderElectionTest, SolrIndexSplitterTest, TestFunctionQuery, TestDynamicFieldResource, PathHierarchyTokenizerFactoryTest, ExternalFileFieldSortTest, HdfsCollectionsAPIDistributedZkTest, TestManagedSchemaFieldResource, DateFieldTest, SliceStateUpdateTest, TestPerFieldSimilarity, QueryResultKeyTest, CopyFieldTest, TestReloadAndDeleteDocs, FileBasedSpellCheckerTest, SuggesterTSTTest, TestBinaryResponseWriter, TestFreeTextSuggestions, TestPseudoReturnFields, PolyFieldTest, BJQParserTest, TestCopyFieldCollectionResource, HighlighterConfigTest, DistribCursorPagingTest, RegexBoostProcessorTest, TestDynamicFieldCollectionResource, LegacyHTMLStripCharFilterTest, PingRequestHandlerTest, TestFoldingMultitermQuery, SuggesterFSTTest, TestBinaryField, TestRangeQuery, TestIndexingPerformance, BlockDirectoryTest, TestUtils, ResourceLoaderTest, TestInitQParser, TestDistributedSearch, SimplePostToolTest, TestManagedSchema, TestFieldTypeCollectionResource, BadIndexSchemaTest, StandardRequestHandlerTest, MigrateRouteKeyTest, XsltUpdateRequestHandlerTest, ConvertedLegacyTest, DirectSolrConnectionTest, TestFileDictionaryLookup, TestStressVersions, BadComponentTest, ShardRoutingTest, NoCacheHeaderTest, TestRandomMergePolicy, BasicDistributedZk2Test, ShardSplitTest, TestNumberUtils, RequiredFieldsTest, TestReplicationHandler, TestNoOpRegenerator, DeleteReplicaTest, TestSearchPerf, CachingDirectoryFactoryTest, TestFastOutputStream, JSONWriterTest, TestBadConfig, MBeansHandlerTest, BasicFunctionalityTest, HdfsChaosMonkeySafeLeaderTest, UniqFieldsUpdateProcessorFactoryTest, TestFiltering, SolrInfoMBeanTest, AssignTest, ChaosMonkeySafeLeaderTest, DistributedTermsComponentTest, TestLFUCache, SolrIndexConfigTest, SpellingQueryConverterTest, TestAddFieldRealTimeGet, URLClassifyProcessorTest, CollectionsAPIDistributedZkTest, BasicZkTest, ChaosMonkeyNothingIsSafeTest, TestRandomFaceting, LeaderElectionIntegrationTest, FullSolrCloudDistribCmdsTest, TestCloudManagedSchema, RemoteQueryErrorTest, OverseerTest]
   [junit4] Completed on J1 in 27.62s, 8 tests, 1 error <<< FAILURES!

[...truncated 505 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1272: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:905: There were test failures: 362 suites, 1563 tests, 1 error, 35 ignored (7 assumptions)

Total time: 76 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1905 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1905/

1 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:40207/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:40207/collection1
	at __randomizedtesting.SeedInfo.seed([A632229E9ABA1A5E:27D4AC86EDE57A62]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:503)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:168)
	at org.apache.solr.client.solrj.SolrServer.commit(SolrServer.java:146)
	at org.apache.solr.cloud.BasicDistributedZkTest.testUpdateProcessorsRunOnlyOnce(BasicDistributedZkTest.java:663)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:150)
	at java.net.SocketInputStream.read(SocketInputStream.java:121)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)
	at sun.security.ssl.InputRecord.read(InputRecord.java:480)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:395)
	... 47 more




Build Log:
[...truncated 10420 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> 130505 T1398 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 130506 T1398 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 130513 T1398 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1392850127119
   [junit4]   2> 130515 T1398 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 130515 T1399 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 130616 T1398 oasc.ZkTestServer.run start zk server on port:48924
   [junit4]   2> 130617 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130622 T1405 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56784481 name:ZooKeeperConnection Watcher:127.0.0.1:48924 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130623 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130623 T1398 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 130636 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130638 T1407 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2630210a name:ZooKeeperConnection Watcher:127.0.0.1:48924/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130638 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 130639 T1398 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 130650 T1398 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 130654 T1398 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 130658 T1398 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 130663 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 130664 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 130678 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 130679 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 130785 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 130786 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 130791 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 130791 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 130796 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 130797 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 130801 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 130802 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 130807 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 130807 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 130812 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 130812 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 130817 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 130818 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 130822 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 130823 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 130827 T1398 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 130828 T1398 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 130834 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 130843 T1409 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c7407d7 name:ZooKeeperConnection Watcher:127.0.0.1:48924/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 130843 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131153 T1398 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 131159 T1398 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 131164 T1398 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:16868
   [junit4]   2> 131168 T1398 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 131168 T1398 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 131169 T1398 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392850127452
   [junit4]   2> 131169 T1398 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392850127452/'
   [junit4]   2> 131208 T1398 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392850127452/solr.xml
   [junit4]   2> 131309 T1398 oasc.CoreContainer.<init> New CoreContainer 1298623353
   [junit4]   2> 131310 T1398 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392850127452/]
   [junit4]   2> 131312 T1398 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 131312 T1398 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 131313 T1398 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 131313 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 131314 T1398 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 131315 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 131315 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 131316 T1398 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 131316 T1398 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 131334 T1398 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 131334 T1398 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 131335 T1398 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 131336 T1398 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48924/solr
   [junit4]   2> 131336 T1398 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 131338 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131341 T1420 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bbe141a name:ZooKeeperConnection Watcher:127.0.0.1:48924 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131342 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131354 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131356 T1422 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31dc66fd name:ZooKeeperConnection Watcher:127.0.0.1:48924/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131357 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131360 T1398 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 131367 T1398 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 131373 T1398 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 131377 T1398 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:16868_
   [junit4]   2> 131379 T1398 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:16868_
   [junit4]   2> 131389 T1398 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 131393 T1398 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 131406 T1398 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:16868_
   [junit4]   2> 131406 T1398 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 131410 T1398 oasc.Overseer.start Overseer (id=91281825931395076-127.0.0.1:16868_-n_0000000000) starting
   [junit4]   2> 131416 T1398 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 131427 T1424 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 131428 T1398 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 131428 T1424 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 131432 T1398 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 131435 T1398 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 131439 T1423 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 131446 T1425 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 131447 T1425 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 131449 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 131449 T1425 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 131451 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 131452 T1423 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:16868",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16868_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 131452 T1423 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 131453 T1423 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 131457 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 131465 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 132450 T1425 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 132450 T1425 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392850127452/collection1
   [junit4]   2> 132450 T1425 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 132452 T1425 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 132452 T1425 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 132454 T1425 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 132454 T1425 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392850127452/collection1/'
   [junit4]   2> 132455 T1425 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392850127452/collection1/lib/README' to classloader
   [junit4]   2> 132456 T1425 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392850127452/collection1/lib/classes/' to classloader
   [junit4]   2> 132527 T1425 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 132615 T1425 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 132717 T1425 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 132732 T1425 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 133427 T1425 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 133428 T1425 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 133429 T1425 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 133444 T1425 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 133449 T1425 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 133477 T1425 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 133483 T1425 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 133489 T1425 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 133491 T1425 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 133491 T1425 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 133491 T1425 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 133493 T1425 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 133493 T1425 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 133494 T1425 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 133495 T1425 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-controljetty-1392850127452/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/control/data/
   [junit4]   2> 133496 T1425 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@53761ed1
   [junit4]   2> 133497 T1425 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/control/data
   [junit4]   2> 133497 T1425 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/control/data/index/
   [junit4]   2> 133498 T1425 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 133498 T1425 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/control/data/index
   [junit4]   2> 133499 T1425 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.41796875, noCFSRatio=1.0]
   [junit4]   2> 133500 T1425 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@43cd41c9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7b1a0bba),segFN=segments_1,generation=1}
   [junit4]   2> 133500 T1425 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 133506 T1425 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 133507 T1425 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 133508 T1425 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 133508 T1425 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 133508 T1425 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 133509 T1425 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 133509 T1425 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 133510 T1425 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 133510 T1425 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 133511 T1425 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 133512 T1425 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 133512 T1425 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 133513 T1425 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 133514 T1425 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 133514 T1425 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 133515 T1425 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 133532 T1425 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 133539 T1425 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 133539 T1425 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 133551 T1425 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=655403911, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 133552 T1425 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@43cd41c9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7b1a0bba),segFN=segments_1,generation=1}
   [junit4]   2> 133552 T1425 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 133553 T1425 oass.SolrIndexSearcher.<init> Opening Searcher@2725445[collection1] main
   [junit4]   2> 133559 T1426 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2725445[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 133562 T1425 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 133563 T1398 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 133564 T1398 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 133563 T1429 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:16868 collection:control_collection shard:shard1
   [junit4]   2> 133567 T1429 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 133570 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133572 T1431 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@495cf668 name:ZooKeeperConnection Watcher:127.0.0.1:48924/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133573 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133585 T1398 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 133590 T1429 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 133590 T1398 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 133593 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 133594 T1429 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 133595 T1429 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C61 name=collection1 org.apache.solr.core.SolrCore@5bdc0e9a url=https://127.0.0.1:16868/collection1 node=127.0.0.1:16868_ C61_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:16868, core=collection1, node_name=127.0.0.1:16868_}
   [junit4]   2> 133595 T1429 C61 P16868 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:16868/collection1/
   [junit4]   2> 133595 T1429 C61 P16868 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 133596 T1429 C61 P16868 oasc.SyncStrategy.syncToMe https://127.0.0.1:16868/collection1/ has no replicas
   [junit4]   2> 133596 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 133596 T1429 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:16868/collection1/ shard1
   [junit4]   2> 133597 T1429 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 133601 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 133605 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133605 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133621 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 133626 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 133765 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133765 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133772 T1429 oasc.ZkController.register We are https://127.0.0.1:16868/collection1/ and leader is https://127.0.0.1:16868/collection1/
   [junit4]   2> 133772 T1429 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:16868
   [junit4]   2> 133772 T1429 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 133773 T1429 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 133773 T1429 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 133806 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 133806 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 133806 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 133807 T1429 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 133809 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 133810 T1423 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:16868",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:16868_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 133849 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 133901 T1398 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 133902 T1398 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 133907 T1398 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 133909 T1398 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:33154
   [junit4]   2> 133912 T1398 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 133913 T1398 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 133913 T1398 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392850130196
   [junit4]   2> 133914 T1398 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392850130196/'
   [junit4]   2> 133953 T1398 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392850130196/solr.xml
   [junit4]   2> 133953 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133953 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 134053 T1398 oasc.CoreContainer.<init> New CoreContainer 1986359499
   [junit4]   2> 134053 T1398 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392850130196/]
   [junit4]   2> 134055 T1398 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 134056 T1398 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 134056 T1398 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 134057 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 134057 T1398 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 134058 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 134058 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 134059 T1398 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 134059 T1398 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 134076 T1398 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 134077 T1398 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 134077 T1398 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 134078 T1398 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48924/solr
   [junit4]   2> 134079 T1398 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 134080 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 134088 T1442 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65134494 name:ZooKeeperConnection Watcher:127.0.0.1:48924 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 134089 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 134097 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 134105 T1444 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@53a3cfef name:ZooKeeperConnection Watcher:127.0.0.1:48924/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 134105 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 134114 T1398 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 135120 T1398 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33154_
   [junit4]   2> 135127 T1398 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33154_
   [junit4]   2> 135132 T1431 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 135132 T1444 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 135132 T1422 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 135151 T1445 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 135151 T1445 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 135153 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 135153 T1445 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 135153 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 135154 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 135156 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 135158 T1423 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:33154",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33154_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 135158 T1423 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 135158 T1423 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 135163 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 135266 T1444 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 135266 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 135266 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 136155 T1445 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 136155 T1445 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392850130196/collection1
   [junit4]   2> 136155 T1445 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 136157 T1445 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 136157 T1445 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 136158 T1445 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 136159 T1445 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392850130196/collection1/'
   [junit4]   2> 136160 T1445 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392850130196/collection1/lib/README' to classloader
   [junit4]   2> 136161 T1445 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392850130196/collection1/lib/classes/' to classloader
   [junit4]   2> 136230 T1445 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 136304 T1445 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 136406 T1445 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 136422 T1445 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 137096 T1445 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 137097 T1445 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 137098 T1445 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 137112 T1445 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 137117 T1445 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 137142 T1445 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 137149 T1445 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 137154 T1445 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 137156 T1445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 137156 T1445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 137157 T1445 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 137158 T1445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 137159 T1445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 137159 T1445 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 137160 T1445 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty1-1392850130196/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty1/
   [junit4]   2> 137160 T1445 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@53761ed1
   [junit4]   2> 137162 T1445 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty1
   [junit4]   2> 137162 T1445 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty1/index/
   [junit4]   2> 137162 T1445 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 137163 T1445 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty1/index
   [junit4]   2> 137163 T1445 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.41796875, noCFSRatio=1.0]
   [junit4]   2> 137165 T1445 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@59318900 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@156e4504),segFN=segments_1,generation=1}
   [junit4]   2> 137165 T1445 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 137173 T1445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 137173 T1445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 137174 T1445 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 137174 T1445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 137174 T1445 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 137175 T1445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 137175 T1445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 137176 T1445 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 137176 T1445 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 137177 T1445 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 137179 T1445 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 137179 T1445 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 137179 T1445 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 137181 T1445 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 137181 T1445 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 137182 T1445 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 137199 T1445 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 137205 T1445 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 137205 T1445 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 137206 T1445 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=655403911, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 137207 T1445 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@59318900 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@156e4504),segFN=segments_1,generation=1}
   [junit4]   2> 137208 T1445 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 137208 T1445 oass.SolrIndexSearcher.<init> Opening Searcher@5bf2d540[collection1] main
   [junit4]   2> 137216 T1446 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5bf2d540[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 137221 T1445 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 137221 T1398 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 137222 T1398 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 137221 T1449 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:33154 collection:collection1 shard:shard2
   [junit4]   2> 137225 T1449 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 137248 T1449 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 137251 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 137251 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 137252 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 137252 T1449 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 137252 T1449 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C62 name=collection1 org.apache.solr.core.SolrCore@54c1224b url=https://127.0.0.1:33154/collection1 node=127.0.0.1:33154_ C62_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:33154, core=collection1, node_name=127.0.0.1:33154_}
   [junit4]   2> 137253 T1449 C62 P33154 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:33154/collection1/
   [junit4]   2> 137253 T1449 C62 P33154 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 137253 T1449 C62 P33154 oasc.SyncStrategy.syncToMe https://127.0.0.1:33154/collection1/ has no replicas
   [junit4]   2> 137254 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 137254 T1449 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:33154/collection1/ shard2
   [junit4]   2> 137254 T1449 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 137259 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 137260 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137261 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137261 T1444 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137267 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 137272 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 137376 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137376 T1444 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137376 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137418 T1449 oasc.ZkController.register We are https://127.0.0.1:33154/collection1/ and leader is https://127.0.0.1:33154/collection1/
   [junit4]   2> 137418 T1449 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:33154
   [junit4]   2> 137418 T1449 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 137419 T1449 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 137419 T1449 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 137421 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 137421 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 137422 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 137422 T1449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 137424 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 137425 T1423 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:33154",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33154_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 137430 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 137534 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137534 T1444 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137534 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137540 T1398 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 137541 T1398 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 137546 T1398 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 137548 T1398 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:17030
   [junit4]   2> 137551 T1398 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 137552 T1398 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 137552 T1398 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392850133830
   [junit4]   2> 137553 T1398 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392850133830/'
   [junit4]   2> 137591 T1398 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392850133830/solr.xml
   [junit4]   2> 137690 T1398 oasc.CoreContainer.<init> New CoreContainer 1779201762
   [junit4]   2> 137691 T1398 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392850133830/]
   [junit4]   2> 137692 T1398 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 137693 T1398 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 137694 T1398 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 137694 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 137695 T1398 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 137695 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 137696 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 137696 T1398 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 137697 T1398 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 137714 T1398 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 137714 T1398 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 137715 T1398 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 137715 T1398 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48924/solr
   [junit4]   2> 137716 T1398 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 137717 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 137724 T1460 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4511bcbb name:ZooKeeperConnection Watcher:127.0.0.1:48924 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 137725 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 137729 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 137732 T1462 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b93ad24 name:ZooKeeperConnection Watcher:127.0.0.1:48924/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 137732 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 137745 T1398 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 138751 T1398 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:17030_
   [junit4]   2> 138753 T1398 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:17030_
   [junit4]   2> 138758 T1444 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 138758 T1462 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 138758 T1431 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 138758 T1422 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 138770 T1463 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 138771 T1463 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 138779 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138779 T1463 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 138779 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138780 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138782 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 138783 T1423 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:17030",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:17030_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 138784 T1423 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 138784 T1423 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 138827 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138931 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 138931 T1444 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 138931 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 138931 T1462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 139781 T1463 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 139781 T1463 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392850133830/collection1
   [junit4]   2> 139782 T1463 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 139783 T1463 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 139783 T1463 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 139785 T1463 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 139785 T1463 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392850133830/collection1/'
   [junit4]   2> 139787 T1463 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392850133830/collection1/lib/README' to classloader
   [junit4]   2> 139788 T1463 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392850133830/collection1/lib/classes/' to classloader
   [junit4]   2> 139867 T1463 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 139940 T1463 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 140042 T1463 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 140057 T1463 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 140739 T1463 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 140739 T1463 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 140740 T1463 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 140755 T1463 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 140759 T1463 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 140785 T1463 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 140791 T1463 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 140797 T1463 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 140799 T1463 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 140799 T1463 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 140800 T1463 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 140801 T1463 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 140802 T1463 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 140802 T1463 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 140803 T1463 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZkTest-jetty2-1392850133830/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty2/
   [junit4]   2> 140803 T1463 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@53761ed1
   [junit4]   2> 140804 T1463 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty2
   [junit4]   2> 140805 T1463 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty2/index/
   [junit4]   2> 140805 T1463 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 140806 T1463 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZkTest-1392850127118/jetty2/index
   [junit4]   2> 140806 T1463 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.41796875, noCFSRatio=1.0]
   [junit4]   2> 140808 T1463 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@729cd85b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@27f7a26a),segFN=segments_1,generation=1}
   [junit4]   2> 140808 T1463 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 140816 T1463 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 140817 T1463 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 140817 T1463 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 140817 T1463 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 140818 T1463 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 140818 T1463 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 140819 T1463 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 140819 T1463 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 140820 T1463 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 140821 T1463 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 140822 T1463 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 140822 T1463 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 140823 T1463 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 140824 T1463 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 140825 T1463 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 140826 T1463 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 140842 T1463 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 140848 T1463 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 140848 T1463 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 140850 T1463 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=655403911, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 140851 T1463 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@729cd85b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@27f7a26a),segFN=segments_1,generation=1}
   [junit4]   2> 140851 T1463 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 140851 T1463 oass.SolrIndexSearcher.<init> Opening Searcher@42f024f1[collection1] main
   [junit4]   2> 140859 T1464 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@42f024f1[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 140863 T1463 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 140864 T1398 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 140865 T1398 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 140864 T1467 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:17030 collection:collection1 shard:shard1
   [junit4]   2> 140868 T1467 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 140894 T1467 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 140897 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140898 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140898 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140898 T1467 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 140899 T1467 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C63 name=collection1 org.apache.solr.core.SolrCore@52b5306 url=https://127.0.0.1:17030/collection1 node=127.0.0.1:17030_ C63_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:17030, core=collection1, node_name=127.0.0.1:17030_}
   [junit4]   2> 140899 T1467 C63 P17030 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:17030/collection1/
   [junit4]   2> 140899 T1467 C63 P17030 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 140900 T1467 C63 P17030 oasc.SyncStrategy.syncToMe https://127.0.0.1:17030/collection1/ has no replicas
   [junit4]   2> 140900 T1467 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:17030/collection1/ shard1
   [junit4]   2> 140900 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 140900 T1467 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 140905 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140907 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 140907 T1444 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 140907 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 140907 T1462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 140913 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 140918 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 141021 T1444 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141021 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141021 T1462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141021 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141063 T1467 oasc.ZkController.register We are https://127.0.0.1:17030/collection1/ and leader is https://127.0.0.1:17030/collection1/
   [junit4]   2> 141063 T1467 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:17030
   [junit4]   2> 141063 T1467 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 141064 T1467 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 141064 T1467 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 141066 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 141067 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 141067 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 141067 T1467 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 141069 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 141071 T1423 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:17030",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:17030_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 141075 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 141175 T1398 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 141176 T1398 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 141182 T1398 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 141182 T1462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141182 T1422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141182 T1444 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141182 T1431 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141184 T1398 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:17600
   [junit4]   2> 141188 T1398 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 141188 T1398 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 141189 T1398 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392850137473
   [junit4]   2> 141189 T1398 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392850137473/'
   [junit4]   2> 141228 T1398 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392850137473/solr.xml
   [junit4]   2> 141325 T1398 oasc.CoreContainer.<init> New CoreContainer 1424966577
   [junit4]   2> 141326 T1398 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZkTest-jetty3-1392850137473/]
   [junit4]   2> 141328 T1398 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 141328 T1398 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 141329 T1398 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 141329 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 141330 T1398 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 141330 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 141331 T1398 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 141331 T1398 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 141332 T1398 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 141349 T1398 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 141349 T1398 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 141350 T1398 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 141350 T1398 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48924/solr
   [junit4]   2> 141351 T1398 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 141352 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 141355 T1478 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@51050034 name:ZooKeeperConnection Watcher:127.0.0.1:48924 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 141356 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 141359 T1398 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 141362 T1480 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2aa13666 name:ZooKeeperConnection Watcher:127.0.0.1:48924/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 141362 T1398 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 141375 T1398 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 142380 T1398 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:17600_
   [junit4]   2> 142382 T1398 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:17600_
   [junit4]   2> 142387 T1462 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142387 T1480 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142387 T1444 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142387 T1431 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142387 T1422 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142399 T1481 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 142400 T1481 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 142402 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 142402 T1481 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 142402 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 142403 T1422 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 142405 T1423 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 142406 T1423 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:17600",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:17600_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 142406 T1423 oasc.Overseer$ClusterStat

[...truncated too long message...]

,transaction_logs_total_size=0,transaction_logs_total_number=1}
   [junit4]   2> 272871 T1500 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 272871 T1500 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 272872 T1500 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 272873 T1500 oasc.SolrCore.closeSearcher [collection2] Closing main searcher on request.
   [junit4]   2> 272874 T1500 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 272874 T1500 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1392850127119/collection24 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1392850127119/collection24;done=false>>]
   [junit4]   2> 272874 T1500 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1392850127119/collection24
   [junit4]   2> 272875 T1500 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1392850127119/collection24/index [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1392850127119/collection24/index;done=false>>]
   [junit4]   2> 272875 T1500 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZkTest-1392850127119/collection24/index
   [junit4]   2> 272875 T1500 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection3/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 272876 T1500 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 272877 T1500 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 272877 T1500 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Lucene46: {name=FST41, cat=PostingsFormat(name=MockSep), regex_dup_A_s=PostingsFormat(name=Lucene41WithOrds), series_t=PostingsFormat(name=NestedPulsing), author_t=PostingsFormat(name=NestedPulsing), sequence_i=PostingsFormat(name=Lucene41WithOrds), other_tl1=PostingsFormat(name=MockSep), foo_b=PostingsFormat(name=Lucene41WithOrds), inStock=PostingsFormat(name=Lucene41WithOrds), range_facet_si=PostingsFormat(name=NestedPulsing), a_si=PostingsFormat(name=NestedPulsing), n_ti1=FST41, range_facet_l=PostingsFormat(name=MockSep), foo_f=PostingsFormat(name=Lucene41WithOrds), n_d1=PostingsFormat(name=MockSep), timestamp=PostingsFormat(name=MockSep), n_dt1=PostingsFormat(name=NestedPulsing), regex_dup_B_s=PostingsFormat(name=MockSep), _version_=PostingsFormat(name=MockSep), intDefault=PostingsFormat(name=MockSep), n_l1=PostingsFormat(name=MockSep), rnd_b=FST41, a_t=PostingsFormat(name=MockSep), n_tdt1=PostingsFormat(name=MockSep), range_facet_sl=PostingsFormat(name=Lucene41WithOrds), oddField_s=PostingsFormat(name=Lucene41WithOrds), n_tf1=PostingsFormat(name=MockSep), price=PostingsFormat(name=Lucene41WithOrds), id=FST41, foo_i=PostingsFormat(name=NestedPulsing), multiDefault=PostingsFormat(name=NestedPulsing), text=PostingsFormat(name=Lucene41WithOrds), n_tl1=PostingsFormat(name=NestedPulsing), foo_d=FST41, genre_s=PostingsFormat(name=Lucene41WithOrds), n_td1=PostingsFormat(name=NestedPulsing), val_i=FST41, n_f1=PostingsFormat(name=NestedPulsing), SubjectTerms_mfacet=PostingsFormat(name=MockSep)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=vi, timezone=Mexico/General
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=1,free=253218528,total=454033408
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsUnloadDistributedZkTest, TestQueryUtils, TestCustomSort, TestDynamicFieldCollectionResource, PreAnalyzedFieldTest, TestSweetSpotSimilarityFactory, UpdateRequestProcessorFactoryTest, ScriptEngineTest, TestUtils, TestLRUCache, RecoveryZkTest, SuggestComponentTest, HighlighterConfigTest, DirectUpdateHandlerOptimizeTest, TestDynamicFieldResource, TestQuerySenderNoQuery, TestNonNRTOpen, BasicDistributedZkTest]
   [junit4] Completed on J0 in 142.76s, 1 test, 1 error <<< FAILURES!

[...truncated 1096 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1272: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:905: There were test failures: 362 suites, 1563 tests, 1 error, 36 ignored (8 assumptions)

Total time: 75 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure