You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/07/09 07:34:51 UTC

[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 2026 - Failure

Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/2026/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1
	at __randomizedtesting.SeedInfo.seed([CBCFCFDB8C8FA971:4A2941C3FBD0C94D]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:545)
	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:350)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11207 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/init-core-data-001
   [junit4]   2> 1495540 T6449 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1495546 T6449 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1495547 T6449 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1495547 T6450 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1495647 T6449 oasc.ZkTestServer.run start zk server on port:14701
   [junit4]   2> 1495648 T6449 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1495652 T6456 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4c3222 name:ZooKeeperConnection Watcher:127.0.0.1:14701 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1495652 T6449 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1495652 T6449 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1495655 T6449 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1495656 T6458 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ac3118 name:ZooKeeperConnection Watcher:127.0.0.1:14701/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1495656 T6449 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1495657 T6449 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1495658 T6449 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1495660 T6449 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1495661 T6449 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1495663 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1495664 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1495666 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1495667 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1495769 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1495769 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1495771 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1495772 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1495774 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1495774 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1495776 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1495777 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1495779 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1495779 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1495781 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1495782 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1495783 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1495784 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1495786 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1495787 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1495788 T6449 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1495789 T6449 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1496861 T6449 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1496865 T6449 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14705
   [junit4]   2> 1496866 T6449 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1496866 T6449 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1496867 T6449 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-002
   [junit4]   2> 1496867 T6449 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-002/'
   [junit4]   2> 1496903 T6449 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-002/solr.xml
   [junit4]   2> 1496959 T6449 oasc.CoreContainer.<init> New CoreContainer 969179268
   [junit4]   2> 1496959 T6449 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-002/]
   [junit4]   2> 1496960 T6449 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1496960 T6449 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1496961 T6449 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1496961 T6449 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1496961 T6449 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1496962 T6449 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1496962 T6449 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1496962 T6449 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1496962 T6449 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1496963 T6449 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1496963 T6449 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1496963 T6449 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1496964 T6449 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14701/solr
   [junit4]   2> 1496964 T6449 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1496965 T6449 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496967 T6469 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13d177bd name:ZooKeeperConnection Watcher:127.0.0.1:14701 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496967 T6449 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1496969 T6449 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1496970 T6471 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58ab6ce4 name:ZooKeeperConnection Watcher:127.0.0.1:14701/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1496970 T6449 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1496972 T6449 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1496974 T6449 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1496976 T6449 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1496978 T6449 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1496979 T6449 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1496981 T6449 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1496983 T6449 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14704_
   [junit4]   2> 1496983 T6449 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14704_
   [junit4]   2> 1496985 T6449 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1496987 T6449 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1496989 T6449 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:14704_
   [junit4]   2> 1496989 T6449 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1496991 T6449 oasc.Overseer.start Overseer (id=92070309105893379-127.0.0.1:14704_-n_0000000000) starting
   [junit4]   2> 1496993 T6449 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1496998 T6473 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1496999 T6449 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1497001 T6449 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1497002 T6449 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1497004 T6472 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1497009 T6474 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1497009 T6474 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1497010 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1497010 T6474 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1497011 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1497011 T6472 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14704",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:14704_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1497011 T6472 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1497012 T6472 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1497013 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1497014 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1498010 T6474 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1498011 T6474 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1498012 T6474 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1498012 T6474 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1498013 T6474 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1498013 T6474 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-002/collection1/'
   [junit4]   2> 1498014 T6474 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1498015 T6474 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1498016 T6474 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1498081 T6474 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1498130 T6474 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1498231 T6474 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1498243 T6474 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1498714 T6474 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1498715 T6474 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1498716 T6474 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1498729 T6474 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1498732 T6474 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1498743 T6474 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1498748 T6474 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1498752 T6474 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1498753 T6474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1498753 T6474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1498753 T6474 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1498754 T6474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1498754 T6474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1498754 T6474 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1498755 T6474 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1498755 T6474 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-002/collection1/, dataDir=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/control/data/
   [junit4]   2> 1498755 T6474 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6134d4a1
   [junit4]   2> 1498756 T6474 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/control/data
   [junit4]   2> 1498756 T6474 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/control/data/index/
   [junit4]   2> 1498756 T6474 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1498757 T6474 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/control/data/index
   [junit4]   2> 1498757 T6474 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=91.859375, floorSegmentMB=1.5, forceMergeDeletesPctAllowed=10.191252271037547, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4283913755192833
   [junit4]   2> 1498758 T6474 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1cf5ae6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@159eb208),segFN=segments_1,generation=1}
   [junit4]   2> 1498758 T6474 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1498762 T6474 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1498763 T6474 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1498763 T6474 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1498763 T6474 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1498764 T6474 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1498764 T6474 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1498764 T6474 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1498765 T6474 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1498765 T6474 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1498766 T6474 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1498766 T6474 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1498766 T6474 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1498767 T6474 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1498767 T6474 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1498767 T6474 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1498768 T6474 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1498786 T6474 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1498792 T6474 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1498792 T6474 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1498793 T6474 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=661724280, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1498794 T6474 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1cf5ae6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@159eb208),segFN=segments_1,generation=1}
   [junit4]   2> 1498794 T6474 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1498794 T6474 oass.SolrIndexSearcher.<init> Opening Searcher@7ac875f3[collection1] main
   [junit4]   2> 1498795 T6474 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1498795 T6474 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1498796 T6474 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1498796 T6474 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1498796 T6474 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1498797 T6474 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1498797 T6474 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1498797 T6474 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1498798 T6474 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1498798 T6474 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1498801 T6475 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7ac875f3[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1498802 T6474 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1498803 T6478 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14704 collection:control_collection shard:shard1
   [junit4]   2> 1498803 T6449 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1498803 T6449 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1498803 T6478 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1498808 T6478 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1498808 T6449 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1498809 T6481 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@bfad3d0 name:ZooKeeperConnection Watcher:127.0.0.1:14701/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1498809 T6449 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1498810 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1498810 T6478 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1498810 T6478 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1498810 T6449 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1498811 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C948 name=collection1 org.apache.solr.core.SolrCore@ebaea8f url=http://127.0.0.1:14704/collection1 node=127.0.0.1:14704_ C948_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:14704, core=collection1, node_name=127.0.0.1:14704_}
   [junit4]   2> 1498810 T6478 C948 P14704 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14704/collection1/
   [junit4]   2> 1498811 T6478 C948 P14704 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1498812 T6478 C948 P14704 oasc.SyncStrategy.syncToMe http://127.0.0.1:14704/collection1/ has no replicas
   [junit4]   2> 1498812 T6478 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14704/collection1/ shard1
   [junit4]   2> 1498812 T6478 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1498812 T6449 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1498813 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1498813 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1498813 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1498816 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1498817 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1498818 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1498920 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1498920 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1498969 T6478 oasc.ZkController.register We are http://127.0.0.1:14704/collection1/ and leader is http://127.0.0.1:14704/collection1/
   [junit4]   2> 1498970 T6478 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14704
   [junit4]   2> 1498970 T6478 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1498970 T6478 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1498970 T6478 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1498971 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1498972 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1498972 T6478 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1498972 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1498973 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1498973 T6472 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14704",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:14704_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1498975 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1499076 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1499076 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1499863 T6449 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1499864 T6449 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1499866 T6449 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14710
   [junit4]   2> 1499867 T6449 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1499867 T6449 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1499867 T6449 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003
   [junit4]   2> 1499868 T6449 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/'
   [junit4]   2> 1499899 T6449 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/solr.xml
   [junit4]   2> 1499953 T6449 oasc.CoreContainer.<init> New CoreContainer 329446590
   [junit4]   2> 1499953 T6449 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/]
   [junit4]   2> 1499954 T6449 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1499954 T6449 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1499954 T6449 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1499955 T6449 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1499955 T6449 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1499955 T6449 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1499955 T6449 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1499956 T6449 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1499956 T6449 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1499956 T6449 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1499957 T6449 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1499957 T6449 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1499957 T6449 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14701/solr
   [junit4]   2> 1499958 T6449 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1499958 T6449 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1499960 T6492 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f71f004 name:ZooKeeperConnection Watcher:127.0.0.1:14701 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1499960 T6449 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1499961 T6449 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1499962 T6494 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21e93e21 name:ZooKeeperConnection Watcher:127.0.0.1:14701/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1499963 T6449 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1499967 T6449 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1500969 T6449 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14709_
   [junit4]   2> 1500970 T6449 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14709_
   [junit4]   2> 1500972 T6494 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1500972 T6471 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1500972 T6481 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1500978 T6495 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1500978 T6495 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1500979 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1500979 T6495 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1500979 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1500979 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1500980 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1500980 T6472 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14709",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:14709_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1500981 T6472 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1500981 T6472 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1500982 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1500983 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1500983 T6494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1500983 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1501979 T6495 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1501980 T6495 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1501981 T6495 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1501981 T6495 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1501982 T6495 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1501982 T6495 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/collection1/'
   [junit4]   2> 1501983 T6495 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1501984 T6495 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1501985 T6495 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1502049 T6495 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1502098 T6495 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1502199 T6495 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1502211 T6495 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1502680 T6495 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1502681 T6495 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1502682 T6495 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1502695 T6495 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1502698 T6495 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1502709 T6495 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1502714 T6495 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1502718 T6495 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1502718 T6495 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1502719 T6495 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1502719 T6495 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1502720 T6495 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1502720 T6495 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1502720 T6495 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1502720 T6495 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1502721 T6495 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/collection1/, dataDir=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty1/
   [junit4]   2> 1502721 T6495 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6134d4a1
   [junit4]   2> 1502722 T6495 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty1
   [junit4]   2> 1502722 T6495 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty1/index/
   [junit4]   2> 1502722 T6495 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1502723 T6495 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty1/index
   [junit4]   2> 1502723 T6495 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=91.859375, floorSegmentMB=1.5, forceMergeDeletesPctAllowed=10.191252271037547, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4283913755192833
   [junit4]   2> 1502724 T6495 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@785c9e0a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@54c8dbf4),segFN=segments_1,generation=1}
   [junit4]   2> 1502724 T6495 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1502729 T6495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1502730 T6495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1502730 T6495 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1502730 T6495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1502730 T6495 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1502731 T6495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1502731 T6495 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1502732 T6495 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1502732 T6495 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1502732 T6495 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1502733 T6495 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1502733 T6495 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1502734 T6495 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1502734 T6495 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1502735 T6495 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1502735 T6495 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1502754 T6495 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1502758 T6495 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1502759 T6495 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1502760 T6495 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=661724280, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1502760 T6495 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@785c9e0a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@54c8dbf4),segFN=segments_1,generation=1}
   [junit4]   2> 1502761 T6495 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1502761 T6495 oass.SolrIndexSearcher.<init> Opening Searcher@7e6b7072[collection1] main
   [junit4]   2> 1502761 T6495 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1502762 T6495 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1502762 T6495 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1502763 T6495 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1502763 T6495 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1502763 T6495 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1502764 T6495 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1502764 T6495 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1502764 T6495 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1502765 T6495 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1502769 T6496 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7e6b7072[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1502771 T6495 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1502772 T6499 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14709 collection:collection1 shard:shard1
   [junit4]   2> 1502772 T6449 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1502773 T6449 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1502773 T6499 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1502777 T6499 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1502779 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1502779 T6499 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1502779 T6499 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C949 name=collection1 org.apache.solr.core.SolrCore@a8dc2a0 url=http://127.0.0.1:14709/collection1 node=127.0.0.1:14709_ C949_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:14709, core=collection1, node_name=127.0.0.1:14709_}
   [junit4]   2> 1502779 T6499 C949 P14709 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14709/collection1/
   [junit4]   2> 1502780 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1502780 T6499 C949 P14709 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1502780 T6499 C949 P14709 oasc.SyncStrategy.syncToMe http://127.0.0.1:14709/collection1/ has no replicas
   [junit4]   2> 1502780 T6499 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14709/collection1/ shard1
   [junit4]   2> 1502781 T6499 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1502781 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1502782 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1502782 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1502782 T6494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1502785 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1502786 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1502888 T6494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1502888 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1502888 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1502937 T6499 oasc.ZkController.register We are http://127.0.0.1:14709/collection1/ and leader is http://127.0.0.1:14709/collection1/
   [junit4]   2> 1502938 T6499 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14709
   [junit4]   2> 1502938 T6499 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1502938 T6499 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1502938 T6499 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1502939 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1502939 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1502939 T6499 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1502940 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1502941 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1502941 T6472 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14709",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:14709_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1502943 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1503044 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1503044 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1503044 T6494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1503852 T6449 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1503854 T6449 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1503856 T6449 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65447
   [junit4]   2> 1503856 T6449 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1503857 T6449 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1503857 T6449 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-004
   [junit4]   2> 1503857 T6449 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-004/'
   [junit4]   2> 1503889 T6449 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-004/solr.xml
   [junit4]   2> 1503942 T6449 oasc.CoreContainer.<init> New CoreContainer 893096074
   [junit4]   2> 1503943 T6449 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-004/]
   [junit4]   2> 1503944 T6449 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1503944 T6449 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1503944 T6449 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1503944 T6449 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1503945 T6449 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1503945 T6449 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1503945 T6449 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1503945 T6449 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1503946 T6449 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1503946 T6449 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1503946 T6449 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1503947 T6449 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1503947 T6449 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14701/solr
   [junit4]   2> 1503947 T6449 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1503948 T6449 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1503949 T6511 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@598e6f53 name:ZooKeeperConnection Watcher:127.0.0.1:14701 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1503950 T6449 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1503952 T6449 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1503952 T6513 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@615637fd name:ZooKeeperConnection Watcher:127.0.0.1:14701/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1503953 T6449 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1503957 T6449 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1504959 T6449 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:13674_
   [junit4]   2> 1504960 T6449 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:13674_
   [junit4]   2> 1504963 T6481 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1504963 T6494 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1504963 T6513 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1504963 T6471 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1504970 T6514 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1504970 T6514 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1504971 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1504971 T6514 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1504971 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1504971 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1504972 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1504973 T6472 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13674",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:13674_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1504973 T6472 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1504973 T6472 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1504975 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1504975 T6494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1504975 T6513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1504975 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1504975 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1505971 T6514 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1505972 T6514 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1505972 T6514 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1505973 T6514 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1505973 T6514 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1505974 T6514 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-004/collection1/'
   [junit4]   2> 1505975 T6514 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1505976 T6514 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1505976 T6514 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1506042 T6514 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1506091 T6514 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1506192 T6514 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1506204 T6514 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1506664 T6514 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1506665 T6514 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1506666 T6514 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1506679 T6514 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1506682 T6514 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1506693 T6514 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1506698 T6514 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1506702 T6514 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1506702 T6514 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1506703 T6514 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1506703 T6514 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1506703 T6514 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1506704 T6514 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1506704 T6514 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1506704 T6514 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1506705 T6514 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-004/collection1/, dataDir=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty2/
   [junit4]   2> 1506705 T6514 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6134d4a1
   [junit4]   2> 1506706 T6514 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty2
   [junit4]   2> 1506706 T6514 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty2/index/
   [junit4]   2> 1506706 T6514 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1506707 T6514 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-001/jetty2/index
   [junit4]   2> 1506707 T6514 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=43, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=91.859375, floorSegmentMB=1.5, forceMergeDeletesPctAllowed=10.191252271037547, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4283913755192833
   [junit4]   2> 1506708 T6514 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@61b4fd46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@373aafe),segFN=segments_1,generation=1}
   [junit4]   2> 1506708 T6514 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1506713 T6514 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1506714 T6514 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1506714 T6514 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1506714 T6514 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1506715 T6514 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1506715 T6514 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1506715 T6514 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1506716 T6514 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1506716 T6514 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1506717 T6514 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1506717 T6514 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1506717 T6514 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1506718 T6514 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1506718 T6514 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1506719 T6514 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1506719 T6514 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1506738 T6514 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1506743 T6514 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1506743 T6514 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1506744 T6514 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=661724280, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1506744 T6514 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@61b4fd46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@373aafe),segFN=segments_1,generation=1}
   [junit4]   2> 1506745 T6514 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1506745 T6514 oass.SolrIndexSearcher.<init> Opening Searcher@6b27a657[collection1] main
   [junit4]   2> 1506745 T6514 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1506746 T6514 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1506746 T6514 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1506747 T6514 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1506747 T6514 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1506747 T6514 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1506748 T6514 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1506748 T6514 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1506748 T6514 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1506749 T6514 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1506752 T6515 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6b27a657[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1506755 T6514 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1506756 T6518 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:13674 collection:collection1 shard:shard2
   [junit4]   2> 1506756 T6449 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1506756 T6449 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1506756 T6518 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1506758 T6449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1506759 T6449 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 30000 for each attempt
   [junit4]   2> 1506759 T6449 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30000
   [junit4]   2> 1506760 T6449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1506760 T6518 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1506762 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1506762 T6518 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1506762 T6518 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C950 name=collection1 org.apache.solr.core.SolrCore@15f9ee6 url=http://127.0.0.1:13674/collection1 node=127.0.0.1:13674_ C950_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:13674, core=collection1, node_name=127.0.0.1:13674_}
   [junit4]   2> 1506763 T6518 C950 P13674 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:13674/collection1/
   [junit4]   2> 1506763 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1506763 T6518 C950 P13674 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1506763 T6518 C950 P13674 oasc.SyncStrategy.syncToMe http://127.0.0.1:13674/collection1/ has no replicas
   [junit4]   2> 1506764 T6518 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:13674/collection1/ shard2
   [junit4]   2> 1506764 T6518 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1506765 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1506765 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1506765 T6494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1506765 T6513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1506765 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1506768 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1506769 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1506872 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1506872 T6494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1506872 T6513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1506872 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1506921 T6518 oasc.ZkController.register We are http://127.0.0.1:13674/collection1/ and leader is http://127.0.0.1:13674/collection1/
   [junit4]   2> 1506922 T6518 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:13674
   [junit4]   2> 1506922 T6518 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1506922 T6518 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1506922 T6518 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1506923 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1506924 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1506924 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1506924 T6518 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1506925 T6472 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1506925 T6472 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:13674",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:13674_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1506927 T6471 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1507028 T6471 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1507028 T6513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1507028 T6481 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1507028 T6494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1507761 T6449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1507762 T6449 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
   [junit4]   2> 1507763 T6479 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=12377,localport=14704], receiveBufferSize:6565888
   [junit4]   2> 1507767 T6479 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=14705,localport=22393], receiveBufferSize=6565888
   [junit4]   2> ASYNC  NEW_CORE C951 name=collection1 org.apache.solr.core.SolrCore@ebaea8f url=http://127.0.0.1:14704/collection1 node=127.0.0.1:14704_ C951_STATE=coll:control_collection core:collection1 props:{state=active, base_url=http://127.0.0.1:14704, core=collection1, node_name=127.0.0.1:14704_, leader=true}
   [junit4]   2> 1507768 T6461 C951 P14704 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1507768 T6461 C951 P14704 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1507769 T6461 C951 P14704 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 1507769 T6461 C951 P14704 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 1507770 T6461 C951 P14704 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&softCommit=false&commit=true&wt=javabin&version=2} {commit=} 0 2
   [junit4]   2> 1507871 T6519 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=11258,localport=13674], receiveBufferSize:6565888
   [junit4]   2> 1507871 T6519 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=65447,localport=32891], receiveBufferSize=6565888
   [junit4]   2> 1507874 T6500 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=57881,localport=14709], receiveBufferSize:6565888
   [junit4]   2> 1507875 T6500 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=14710,localport=57724], receiveBufferSize=6565888
   [junit4]   2> ASYNC  NEW_CORE C952 name=collection1 org.apache.solr.core.SolrCore@a8dc2a0 url=http://127.0.0.1:14709/collection1 node=127.0.0.1:14709_ C952_STATE=coll:collection1 core:collection1 props:{state=active, base_url=http://127.0.0.1:14709, core=collection1, node_name=127.0.0.1:14709_, leader=true}
   [junit4]   2> 1507876 T6484 C952 P14709 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1507876 T6484 C952 P14709 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1507877 T6484 C952 P14709 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 1507877 T6484 C952 P14709 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 1507877 T6484 C952 P14709 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&softCommit=false&commit=true&wt=javabin&openSearcher=true&distrib.from=http://127.0.0.1:13674/collection1/&version=2&update.distrib=FROMLEADER&expungeDeletes=false&commit_end_point=true} {commit=} 0 1
   [junit4]   2> 1507978 T6519 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=27652,localport=13674], receiveBufferSize:6565888
   [junit4]   2> 1507979 T6519 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=65447,localport=58923], receiveBufferSize=6565888
   [junit4]   2>  C950_STATE=coll:collection1 core:collection1 props:{state=active, base_url=http://127.0.0.1:13674, core=collection1, node_name=127.0.0.1:13674_, leader=true}
   [junit4]   2> 1507979 T6504 C950 P13674 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1507980 T6504 C950 P13674 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1507980 T6504 C950 P13674 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearche

[...truncated too long message...]

asc.SolrCore.closeSearcher [c8n_1x3_lf_shard1_replica3] Closing main searcher on request.
   [junit4]   2> 2024440 T6494 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2024440 T6494 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/c8n_1x3_lf_shard1_replica3/data/index [CachedDir<<refCount=0;path=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/c8n_1x3_lf_shard1_replica3/data/index;done=false>>]
   [junit4]   2> 2024441 T6494 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/c8n_1x3_lf_shard1_replica3/data/index
   [junit4]   2> 2024441 T6494 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/c8n_1x3_lf_shard1_replica3/data [CachedDir<<refCount=0;path=./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/c8n_1x3_lf_shard1_replica3/data;done=false>>]
   [junit4]   2> 2024441 T6494 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001/tempDir-003/c8n_1x3_lf_shard1_replica3/data
   [junit4]   2> 2024441 T6494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2024442 T6494 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2024442 T6494 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 2024443 T6494 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2024477 T6449 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 13674
   [junit4]   2> 2024478 T6449 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 2024479 T6449 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:14701 14701
   [junit4]   2> 2024544 T6450 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:14701 14701
   [junit4]   2> 2024545 T6449 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:14704/
   [junit4]   2> 2024545 T6449 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:14709/
   [junit4]   2> 2024546 T6449 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:13674/
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HttpPartitionTest -Dtests.method=testDistribSearch -Dtests.seed=CBCFCFDB8C8FA971 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=fr_LU -Dtests.timezone=Pacific/Efate -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    529s J1 | HttpPartitionTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CBCFCFDB8C8FA971:4A2941C3FBD0C94D]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:545)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:350)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2024572 T6449 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-CBCFCFDB8C8FA971-001
   [junit4]   2> 529037 T6448 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene49: {text=Lucene41(blocksize=128), intDefault=PostingsFormat(name=FSTPulsing41), a_t=PostingsFormat(name=MockSep), range_facet_l=PostingsFormat(name=MockSep), range_facet_sl=FST41, id=FST41, _version_=PostingsFormat(name=FSTPulsing41), timestamp=PostingsFormat(name=MockSep), multiDefault=PostingsFormat(name=MockSep), range_facet_si=PostingsFormat(name=MockSep)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=fr_LU, timezone=Pacific/Efate
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=105361024,total=535822336
   [junit4]   2> NOTE: All tests run in this JVM: [ParsingFieldUpdateProcessorsTest, BasicDistributedZk2Test, TestUtils, TestBinaryResponseWriter, XmlUpdateRequestHandlerTest, DeleteShardTest, FullSolrCloudDistribCmdsTest, DistributedDebugComponentTest, TestDistributedGrouping, TestCollationKeyRangeQueries, SuggesterFSTTest, FastVectorHighlighterTest, DateFieldTest, UniqFieldsUpdateProcessorFactoryTest, CursorPagingTest, TestDocSet, SolrRequestParserTest, OverseerCollectionProcessorTest, AddSchemaFieldsUpdateProcessorFactoryTest, AssignTest, TestFieldSortValues, FieldMutatingUpdateProcessorTest, ScriptEngineTest, ClusterStateTest, TestUniqueKeyFieldResource, SuggesterTest, TestExceedMaxTermLength, StatelessScriptUpdateProcessorFactoryTest, TestCollapseQParserPlugin, SolrCoreTest, CoreAdminCreateDiscoverTest, IndexSchemaRuntimeFieldTest, BlockCacheTest, HdfsWriteToMultipleCollectionsTest, TestOverriddenPrefixQueryForCustomFieldType, TestArbitraryIndexDir, TestSolr4Spatial, EchoParamsTest, TestDocumentBuilder, DistributedQueryElevationComponentTest, ChangedSchemaMergeTest, ChaosMonkeySafeLeaderTest, TestRecoveryHdfs, HdfsLockFactoryTest, TestSolrQueryParserDefaultOperatorResource, TestQuerySenderNoQuery, TestRealTimeGet, TestSolrCoreProperties, UpdateRequestProcessorFactoryTest, HdfsCollectionsAPIDistributedZkTest, TestManagedSynonymFilterFactory, HdfsSyncSliceTest, CurrencyFieldXmlFileTest, TestSolrXmlPersistor, TriLevelCompositeIdRoutingTest, TestSolrDeletionPolicy1, TestRequestStatusCollectionAPI, DistributedSuggestComponentTest, PreAnalyzedFieldTest, TestPseudoReturnFields, TestIndexingPerformance, TestDocBasedVersionConstraints, TestNonDefinedSimilarityFactory, QueryResultKeyTest, BlockDirectoryTest, SortByFunctionTest, TestFileDictionaryLookup, TermVectorComponentTest, DistributedExpandComponentTest, TermsComponentTest, TestRandomDVFaceting, DOMUtilTest, TestCustomSort, TestWriterPerf, CollectionsAPIDistributedZkTest, TestSolrXmlPersistence, RankQueryTest, TestCSVLoader, EnumFieldTest, TestNumberUtils, DocValuesTest, TestBinaryField, DefaultValueUpdateProcessorTest, TestReload, DistributedSpellCheckComponentTest, TestHighlightDedupGrouping, XsltUpdateRequestHandlerTest, TestPartialUpdateDeduplication, PeerSyncTest, ZkCLITest, SimplePostToolTest, TestManagedSchema, WordBreakSolrSpellCheckerTest, HttpPartitionTest]
   [junit4] Completed on J1 in 529.58s, 1 test, 1 error <<< FAILURES!

[...truncated 876 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:467: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:923: There were test failures: 406 suites, 1658 tests, 1 error, 51 ignored (17 assumptions)

Total time: 109 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-4.x-Java7 #2025
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 174165783 bytes
Compression is 0.0%
Took 30 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 2031 - Still Failing

Posted by Chris Hostetter <ho...@fucit.org>.
: 2) chain that broke involves WDF which has historically been known to 
: be problematic -- but in theory LUCENE-3843 fixed WDF already?

cut/paste error -- i ment LUCENE-5111


-Hoss
http://www.lucidworks.com/

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 2031 - Still Failing

Posted by Chris Hostetter <ho...@fucit.org>.
NOTE:

1) seed reproduces for me on branch_4x, linux 64bit, java7

2) chain that broke involves WDF which has historically been known to 
be problematic -- but in theory LUCENE-3843 fixed WDF already?

ant test  -Dtestcase=TestRandomChains -Dtests.method=testRandomChains 
-Dtests.seed=2871E466A5044906 -Dtests.multiplier=3 -Dtests.slow=true 
-Dtests.locale=no_NO -Dtests.timezone=Atlantic/South_Georgia 
-Dtests.file.encoding=US-ASCII

Perhaps the problem is specific to the combination of of filters?

   [junit4]   2> TEST FAIL: useCharFilter=false text='v|p({ </ mc '
   [junit4]   2> Exception from random analyzer: 
   [junit4]   2> charfilters=
   [junit4]   2>   
org.apache.lucene.analysis.fa.PersianCharFilter(java.io.StringReader@7cc398b8)
   [junit4]   2>   
org.apache.lucene.analysis.fa.PersianCharFilter(org.apache.lucene.analysis.fa.PersianCharFilter@7ef5b8c5)
   [junit4]   2> tokenizer=
   [junit4]   2>   
org.apache.lucene.analysis.path.PathHierarchyTokenizer(org.apache.lucene.analysis.core.TestRandomChains$CheckThatYouDidntReadAnythingReaderWrapper@690c7d5)
   [junit4]   2> filters=
   [junit4]   2>   
org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter(LUCENE_4_10, 
ValidatingTokenFilter@587d7793 
term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, 
7, [vxaqklts, arraatqvg])
   [junit4]   2>   
org.apache.lucene.analysis.shingle.ShingleFilter(ValidatingTokenFilter@6bbaa8d8 
term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word)
   [junit4]   2> offsetsAreCorrect=false




: Date: Tue, 15 Jul 2014 12:42:31 +0000 (UTC)
: From: Apache Jenkins Server <je...@builds.apache.org>
: Reply-To: dev@lucene.apache.org
: To: dev@lucene.apache.org
: Subject: [JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 2031 - Still Failing
: 
: Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/2031/
: 
: 1 tests failed.
: REGRESSION:  org.apache.lucene.analysis.core.TestRandomChains.testRandomChains
: 
: Error Message:
: startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=2,endOffset=1
: 
: Stack Trace:
: java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=2,endOffset=1
: 	at __randomizedtesting.SeedInfo.seed([2871E466A5044906:1590CD07E21654C6]:0)
: 	at org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl.setOffset(PackedTokenAttributeImpl.java:107)
: 	at org.apache.lucene.analysis.shingle.ShingleFilter.incrementToken(ShingleFilter.java:345)
: 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
: 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:703)
: 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:614)
: 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:513)
: 	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChains(TestRandomChains.java:927)
: 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
: 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
: 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
: 	at java.lang.reflect.Method.invoke(Method.java:606)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
: 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
: 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
: 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
: 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
: 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
: 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
: 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
: 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
: 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
: 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
: 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
: 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
: 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
: 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
: 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
: 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
: 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
: 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
: 	at java.lang.Thread.run(Thread.java:745)
: 
: 
: 
: 
: Build Log:
: [...truncated 5713 lines...]
:    [junit4] Suite: org.apache.lucene.analysis.core.TestRandomChains
:    [junit4]   2> TEST FAIL: useCharFilter=false text='v|p({ </ mc '
:    [junit4]   2> Exception from random analyzer: 
:    [junit4]   2> charfilters=
:    [junit4]   2>   org.apache.lucene.analysis.fa.PersianCharFilter(java.io.StringReader@6f1b57ec)
:    [junit4]   2>   org.apache.lucene.analysis.fa.PersianCharFilter(org.apache.lucene.analysis.fa.PersianCharFilter@188e9d0a)
:    [junit4]   2> tokenizer=
:    [junit4]   2>   org.apache.lucene.analysis.path.PathHierarchyTokenizer(org.apache.lucene.analysis.core.TestRandomChains$CheckThatYouDidntReadAnythingReaderWrapper@576d07c8)
:    [junit4]   2> filters=
:    [junit4]   2>   org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter(LUCENE_4_10, ValidatingTokenFilter@e1796cc term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, 7, [vxaqklts, arraatqvg])
:    [junit4]   2>   org.apache.lucene.analysis.shingle.ShingleFilter(ValidatingTokenFilter@6cc1b6d2 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word)
:    [junit4]   2> offsetsAreCorrect=false
:    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestRandomChains -Dtests.method=testRandomChains -Dtests.seed=2871E466A5044906 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=no_NO -Dtests.timezone=Atlantic/South_Georgia -Dtests.file.encoding=US-ASCII
:    [junit4] ERROR   28.7s J0 | TestRandomChains.testRandomChains <<<
:    [junit4]    > Throwable #1: java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=2,endOffset=1
:    [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2871E466A5044906:1590CD07E21654C6]:0)
:    [junit4]    > 	at org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl.setOffset(PackedTokenAttributeImpl.java:107)
:    [junit4]    > 	at org.apache.lucene.analysis.shingle.ShingleFilter.incrementToken(ShingleFilter.java:345)
:    [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
:    [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:703)
:    [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:614)
:    [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:513)
:    [junit4]    > 	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChains(TestRandomChains.java:927)
:    [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
:    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/build/analysis/common/test/J0/./lucene.analysis.core.TestRandomChains-2871E466A5044906-001
:    [junit4]   2> NOTE: test params are: codec=Lucene49: {dummy=PostingsFormat(name=Lucene41WithOrds)}, docValues:{}, sim=DefaultSimilarity, locale=no_NO, timezone=Atlantic/South_Georgia
:    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=224694112,total=318767104
:    [junit4]   2> NOTE: All tests run in this JVM: [TestHunspellStemFilter, TestGermanLightStemFilterFactory, TestDependencies, TestSoraniAnalyzer, TestCondition2, TestTurkishLowerCaseFilter, TestCzechStemmer, TestMorphData, TestIrishAnalyzer, TestHyphenatedWordsFilter, TestRussianFilters, TestEnglishMinimalStemFilterFactory, TestCapitalizationFilterFactory, TestFlagLong, TestRollingCharBuffer, TestSpanishLightStemFilterFactory, TestSoraniStemFilterFactory, TestCharTokenizers, TestFilesystemResourceLoader, TestEscaped, TestAnalyzers, TestSnowballVocab, TestSpanishLightStemFilter, TestSingleTokenTokenFilter, TestFrenchMinimalStemFilter, TestPatternCaptureGroupTokenFilter, TestPorterStemFilter, TestCharArraySet, TestHunspellStemFilterFactory, TestSnowballPorterFilterFactory, TestTurkishLowerCaseFilterFactory, QueryAutoStopWordAnalyzerTest, TestPortugueseMinimalStemFilterFactory, EdgeNGramTokenizerTest, TestCapitalizationFilter, TestPatternReplaceFilterFactory, TestPatternTokenizerFact
 ory, TestSynonymMapFilter, PositionFilterTest, TestHindiFilters, TestGermanStemFilter, TestZeroAffix, TestIrishLowerCaseFilterFactory, TestNorwegianAnalyzer, TestUAX29URLEmailTokenizerFactory, TestSoraniNormalizationFilter, TestItalianLightStemFilter, TestMappingCharFilter, TestKeywordMarkerFilter, TestLatvianStemmer, TestFullStrip, TestDictionary, TestSoraniNormalizationFilterFactory, TestLimitTokenCountFilterFactory, TestThaiAnalyzer, TestSwedishAnalyzer, TestSolrSynonymParser, TestCJKWidthFilter, TestIndonesianAnalyzer, TestTypeTokenFilter, TestCodepointCountFilterFactory, TestHindiNormalizer, TestEnglishAnalyzer, TestTwoFold, TestPortugueseAnalyzer, TestThaiTokenizerFactory, TestDutchStemmer, TestKStemmer, TestIgnore, TestRandomChains]
:    [junit4] Completed on J0 in 51.07s, 2 tests, 1 error <<< FAILURES!
: 
: [...truncated 406 lines...]
: BUILD FAILED
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:467: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:447: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:45: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/build.xml:544: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:2019: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/analysis/build.xml:106: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/analysis/build.xml:38: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/module-build.xml:60: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1299: The following error occurred while executing this line:
: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:923: There were test failures: 274 suites, 1459 tests, 1 error, 1 ignored
: 
: Total time: 27 minutes 37 seconds
: Build step 'Invoke Ant' marked build as failure
: Archiving artifacts
: Sending artifact delta relative to Lucene-Solr-Tests-4.x-Java7 #2025
: Archived 3 artifacts
: Archive block size is 32768
: Received 0 blocks and 1759320 bytes
: Compression is 0.0%
: Took 0.49 sec
: Recording test results
: Email was triggered for: Failure
: Sending email for trigger: Failure
: 
: 
: 

-Hoss
http://www.lucidworks.com/

[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 2031 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/2031/

1 tests failed.
REGRESSION:  org.apache.lucene.analysis.core.TestRandomChains.testRandomChains

Error Message:
startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=2,endOffset=1

Stack Trace:
java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=2,endOffset=1
	at __randomizedtesting.SeedInfo.seed([2871E466A5044906:1590CD07E21654C6]:0)
	at org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl.setOffset(PackedTokenAttributeImpl.java:107)
	at org.apache.lucene.analysis.shingle.ShingleFilter.incrementToken(ShingleFilter.java:345)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:703)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:614)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:513)
	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChains(TestRandomChains.java:927)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 5713 lines...]
   [junit4] Suite: org.apache.lucene.analysis.core.TestRandomChains
   [junit4]   2> TEST FAIL: useCharFilter=false text='v|p({ </ mc '
   [junit4]   2> Exception from random analyzer: 
   [junit4]   2> charfilters=
   [junit4]   2>   org.apache.lucene.analysis.fa.PersianCharFilter(java.io.StringReader@6f1b57ec)
   [junit4]   2>   org.apache.lucene.analysis.fa.PersianCharFilter(org.apache.lucene.analysis.fa.PersianCharFilter@188e9d0a)
   [junit4]   2> tokenizer=
   [junit4]   2>   org.apache.lucene.analysis.path.PathHierarchyTokenizer(org.apache.lucene.analysis.core.TestRandomChains$CheckThatYouDidntReadAnythingReaderWrapper@576d07c8)
   [junit4]   2> filters=
   [junit4]   2>   org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter(LUCENE_4_10, ValidatingTokenFilter@e1796cc term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, 7, [vxaqklts, arraatqvg])
   [junit4]   2>   org.apache.lucene.analysis.shingle.ShingleFilter(ValidatingTokenFilter@6cc1b6d2 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word)
   [junit4]   2> offsetsAreCorrect=false
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestRandomChains -Dtests.method=testRandomChains -Dtests.seed=2871E466A5044906 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=no_NO -Dtests.timezone=Atlantic/South_Georgia -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   28.7s J0 | TestRandomChains.testRandomChains <<<
   [junit4]    > Throwable #1: java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=2,endOffset=1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2871E466A5044906:1590CD07E21654C6]:0)
   [junit4]    > 	at org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl.setOffset(PackedTokenAttributeImpl.java:107)
   [junit4]    > 	at org.apache.lucene.analysis.shingle.ShingleFilter.incrementToken(ShingleFilter.java:345)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:703)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:614)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:513)
   [junit4]    > 	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChains(TestRandomChains.java:927)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/build/analysis/common/test/J0/./lucene.analysis.core.TestRandomChains-2871E466A5044906-001
   [junit4]   2> NOTE: test params are: codec=Lucene49: {dummy=PostingsFormat(name=Lucene41WithOrds)}, docValues:{}, sim=DefaultSimilarity, locale=no_NO, timezone=Atlantic/South_Georgia
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=224694112,total=318767104
   [junit4]   2> NOTE: All tests run in this JVM: [TestHunspellStemFilter, TestGermanLightStemFilterFactory, TestDependencies, TestSoraniAnalyzer, TestCondition2, TestTurkishLowerCaseFilter, TestCzechStemmer, TestMorphData, TestIrishAnalyzer, TestHyphenatedWordsFilter, TestRussianFilters, TestEnglishMinimalStemFilterFactory, TestCapitalizationFilterFactory, TestFlagLong, TestRollingCharBuffer, TestSpanishLightStemFilterFactory, TestSoraniStemFilterFactory, TestCharTokenizers, TestFilesystemResourceLoader, TestEscaped, TestAnalyzers, TestSnowballVocab, TestSpanishLightStemFilter, TestSingleTokenTokenFilter, TestFrenchMinimalStemFilter, TestPatternCaptureGroupTokenFilter, TestPorterStemFilter, TestCharArraySet, TestHunspellStemFilterFactory, TestSnowballPorterFilterFactory, TestTurkishLowerCaseFilterFactory, QueryAutoStopWordAnalyzerTest, TestPortugueseMinimalStemFilterFactory, EdgeNGramTokenizerTest, TestCapitalizationFilter, TestPatternReplaceFilterFactory, TestPatternTokenizerFactory, TestSynonymMapFilter, PositionFilterTest, TestHindiFilters, TestGermanStemFilter, TestZeroAffix, TestIrishLowerCaseFilterFactory, TestNorwegianAnalyzer, TestUAX29URLEmailTokenizerFactory, TestSoraniNormalizationFilter, TestItalianLightStemFilter, TestMappingCharFilter, TestKeywordMarkerFilter, TestLatvianStemmer, TestFullStrip, TestDictionary, TestSoraniNormalizationFilterFactory, TestLimitTokenCountFilterFactory, TestThaiAnalyzer, TestSwedishAnalyzer, TestSolrSynonymParser, TestCJKWidthFilter, TestIndonesianAnalyzer, TestTypeTokenFilter, TestCodepointCountFilterFactory, TestHindiNormalizer, TestEnglishAnalyzer, TestTwoFold, TestPortugueseAnalyzer, TestThaiTokenizerFactory, TestDutchStemmer, TestKStemmer, TestIgnore, TestRandomChains]
   [junit4] Completed on J0 in 51.07s, 2 tests, 1 error <<< FAILURES!

[...truncated 406 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:467: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/build.xml:544: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:2019: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/analysis/build.xml:106: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/analysis/build.xml:38: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/module-build.xml:60: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:923: There were test failures: 274 suites, 1459 tests, 1 error, 1 ignored

Total time: 27 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-4.x-Java7 #2025
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 1759320 bytes
Compression is 0.0%
Took 0.49 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 2030 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/2030/

1 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1
	at __randomizedtesting.SeedInfo.seed([F3CBDBF4B95B6901:722D55ECCE04093D]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:545)
	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:350)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.GeneratedMethodAccessor46.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11407 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/init-core-data-001
   [junit4]   2> 2212163 T6563 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /stu/
   [junit4]   2> 2212169 T6563 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 2212170 T6563 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2212171 T6564 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2212272 T6563 oasc.ZkTestServer.run start zk server on port:38634
   [junit4]   2> 2212273 T6563 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2212277 T6570 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@748bb834 name:ZooKeeperConnection Watcher:127.0.0.1:38634 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2212277 T6563 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2212277 T6563 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2212281 T6563 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2212282 T6572 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6318d762 name:ZooKeeperConnection Watcher:127.0.0.1:38634/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2212282 T6563 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2212282 T6563 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2212284 T6563 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2212286 T6563 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2212288 T6563 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2212290 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2212290 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2212293 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2212294 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2212397 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2212398 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2212400 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2212401 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2212403 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2212403 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2212405 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2212406 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2212408 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2212409 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 2212411 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2212412 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2212414 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2212414 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2212416 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2212417 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2212419 T6563 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2212420 T6563 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2213556 T6563 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2213561 T6563 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:49750
   [junit4]   2> 2213561 T6563 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2213561 T6563 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2213562 T6563 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-002
   [junit4]   2> 2213562 T6563 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-002/'
   [junit4]   2> 2213593 T6563 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-002/solr.xml
   [junit4]   2> 2213646 T6563 oasc.CoreContainer.<init> New CoreContainer 1899771350
   [junit4]   2> 2213647 T6563 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-002/]
   [junit4]   2> 2213648 T6563 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2213648 T6563 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2213648 T6563 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2213649 T6563 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2213649 T6563 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2213649 T6563 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2213649 T6563 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2213650 T6563 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2213650 T6563 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2213650 T6563 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2213651 T6563 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2213651 T6563 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2213651 T6563 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38634/solr
   [junit4]   2> 2213651 T6563 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2213652 T6563 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2213654 T6583 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27f2aeae name:ZooKeeperConnection Watcher:127.0.0.1:38634 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2213654 T6563 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2213656 T6563 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2213657 T6585 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60e29c5e name:ZooKeeperConnection Watcher:127.0.0.1:38634/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2213657 T6563 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2213658 T6563 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2213660 T6563 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2213662 T6563 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 2213664 T6563 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 2213666 T6563 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 2213668 T6563 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2213669 T6563 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48305_stu
   [junit4]   2> 2213670 T6563 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48305_stu
   [junit4]   2> 2213672 T6563 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 2213673 T6563 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2213675 T6563 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48305_stu
   [junit4]   2> 2213676 T6563 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2213677 T6563 oasc.Overseer.start Overseer (id=92096395758141443-127.0.0.1:48305_stu-n_0000000000) starting
   [junit4]   2> 2213679 T6563 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2213685 T6587 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2213685 T6563 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2213686 T6563 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2213687 T6563 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2213689 T6586 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2213694 T6588 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 2213694 T6588 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2213695 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2213695 T6588 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2213696 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2213697 T6586 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48305/stu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48305_stu",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2213697 T6586 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 2213698 T6586 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2213699 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2213700 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2214697 T6588 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2214698 T6588 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 2214699 T6588 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2214699 T6588 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2214700 T6588 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2214700 T6588 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-002/collection1/'
   [junit4]   2> 2214702 T6588 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 2214702 T6588 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 2214703 T6588 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 2214769 T6588 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 2214818 T6588 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2214920 T6588 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2214931 T6588 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2215381 T6588 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2215382 T6588 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2215383 T6588 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2215396 T6588 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2215399 T6588 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2215411 T6588 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2215415 T6588 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2215419 T6588 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2215420 T6588 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2215420 T6588 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2215421 T6588 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2215421 T6588 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2215422 T6588 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2215422 T6588 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 2215422 T6588 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2215423 T6588 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-002/collection1/, dataDir=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/control/data/
   [junit4]   2> 2215423 T6588 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ca3baae
   [junit4]   2> 2215424 T6588 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/control/data
   [junit4]   2> 2215424 T6588 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/control/data/index/
   [junit4]   2> 2215424 T6588 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2215424 T6588 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/control/data/index
   [junit4]   2> 2215425 T6588 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=39.0751953125, floorSegmentMB=1.1455078125, forceMergeDeletesPctAllowed=3.3525967927110303, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3165725400804661
   [junit4]   2> 2215426 T6588 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d7bcff4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bba370b),segFN=segments_1,generation=1}
   [junit4]   2> 2215426 T6588 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2215430 T6588 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2215430 T6588 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2215430 T6588 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2215431 T6588 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2215431 T6588 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2215431 T6588 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2215432 T6588 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2215432 T6588 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2215432 T6588 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2215433 T6588 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2215433 T6588 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2215434 T6588 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2215434 T6588 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2215434 T6588 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2215435 T6588 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2215435 T6588 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2215454 T6588 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2215460 T6588 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2215460 T6588 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2215461 T6588 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=50.056640625, floorSegmentMB=0.8681640625, forceMergeDeletesPctAllowed=4.029527315893077, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.85939646723903
   [junit4]   2> 2215461 T6588 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5d7bcff4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5bba370b),segFN=segments_1,generation=1}
   [junit4]   2> 2215461 T6588 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2215462 T6588 oass.SolrIndexSearcher.<init> Opening Searcher@ee037f8[collection1] main
   [junit4]   2> 2215462 T6588 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2215463 T6588 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2215463 T6588 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2215464 T6588 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2215464 T6588 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2215464 T6588 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2215465 T6588 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2215465 T6588 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2215465 T6588 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 2215466 T6588 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2215469 T6589 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ee037f8[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2215470 T6588 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2215471 T6592 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48305/stu collection:control_collection shard:shard1
   [junit4]   2> 2215471 T6563 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 2215471 T6563 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2215471 T6592 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 2215476 T6563 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2215476 T6592 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2215477 T6595 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3570076 name:ZooKeeperConnection Watcher:127.0.0.1:38634/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2215477 T6563 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2215478 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215478 T6592 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2215478 T6563 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2215479 T6592 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2215479 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C1527 name=collection1 org.apache.solr.core.SolrCore@46934841 url=http://127.0.0.1:48305/stu/collection1 node=127.0.0.1:48305_stu C1527_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:48305/stu, core=collection1, node_name=127.0.0.1:48305_stu}
   [junit4]   2> 2215479 T6592 C1527 P48305 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48305/stu/collection1/
   [junit4]   2> 2215480 T6592 C1527 P48305 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2215480 T6592 C1527 P48305 oasc.SyncStrategy.syncToMe http://127.0.0.1:48305/stu/collection1/ has no replicas
   [junit4]   2> 2215480 T6592 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48305/stu/collection1/ shard1
   [junit4]   2> 2215480 T6563 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2215480 T6592 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 2215481 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215482 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2215482 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2215485 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215486 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2215488 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215591 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2215591 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2215640 T6592 oasc.ZkController.register We are http://127.0.0.1:48305/stu/collection1/ and leader is http://127.0.0.1:48305/stu/collection1/
   [junit4]   2> 2215641 T6592 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48305/stu
   [junit4]   2> 2215641 T6592 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2215641 T6592 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 2215641 T6592 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2215642 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215642 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215643 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215643 T6592 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2215644 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2215644 T6586 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48305/stu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48305_stu",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 2215646 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2215749 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2215749 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2216714 T6563 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 2216715 T6563 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2216718 T6563 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:14016
   [junit4]   2> 2216719 T6563 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2216720 T6563 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2216720 T6563 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003
   [junit4]   2> 2216720 T6563 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/'
   [junit4]   2> 2216768 T6563 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/solr.xml
   [junit4]   2> 2216849 T6563 oasc.CoreContainer.<init> New CoreContainer 608829417
   [junit4]   2> 2216850 T6563 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/]
   [junit4]   2> 2216851 T6563 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2216851 T6563 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2216852 T6563 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2216852 T6563 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2216853 T6563 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2216853 T6563 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2216853 T6563 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2216853 T6563 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2216854 T6563 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2216854 T6563 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2216854 T6563 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2216855 T6563 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2216855 T6563 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38634/solr
   [junit4]   2> 2216855 T6563 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2216856 T6563 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2216858 T6606 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76402c9b name:ZooKeeperConnection Watcher:127.0.0.1:38634 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2216858 T6563 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2216860 T6563 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2216861 T6608 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7470e907 name:ZooKeeperConnection Watcher:127.0.0.1:38634/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2216861 T6563 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2216866 T6563 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2217870 T6563 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22675_stu
   [junit4]   2> 2217871 T6563 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22675_stu
   [junit4]   2> 2217873 T6585 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2217873 T6608 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2217873 T6595 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2217880 T6609 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 2217881 T6609 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2217881 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2217881 T6609 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2217882 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2217882 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2217883 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2217883 T6586 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22675/stu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22675_stu",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2217884 T6586 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 2217884 T6586 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2217886 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2217886 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2217886 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2217886 T6608 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2218882 T6609 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2218883 T6609 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2218883 T6609 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2218884 T6609 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2218884 T6609 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2218885 T6609 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/collection1/'
   [junit4]   2> 2218886 T6609 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 2218887 T6609 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 2218887 T6609 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 2218953 T6609 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 2219001 T6609 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2219104 T6609 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2219115 T6609 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2219577 T6609 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2219578 T6609 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2219579 T6609 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2219591 T6609 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2219595 T6609 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2219607 T6609 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2219611 T6609 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2219615 T6609 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2219616 T6609 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2219617 T6609 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2219617 T6609 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2219618 T6609 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2219618 T6609 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2219618 T6609 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2219618 T6609 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2219619 T6609 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/collection1/, dataDir=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty1/
   [junit4]   2> 2219619 T6609 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ca3baae
   [junit4]   2> 2219620 T6609 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty1
   [junit4]   2> 2219620 T6609 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty1/index/
   [junit4]   2> 2219620 T6609 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 2219621 T6609 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty1/index
   [junit4]   2> 2219621 T6609 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=39.0751953125, floorSegmentMB=1.1455078125, forceMergeDeletesPctAllowed=3.3525967927110303, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3165725400804661
   [junit4]   2> 2219622 T6609 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@55787a0b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@390d885),segFN=segments_1,generation=1}
   [junit4]   2> 2219622 T6609 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2219627 T6609 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2219627 T6609 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2219627 T6609 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2219628 T6609 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2219628 T6609 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2219628 T6609 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2219629 T6609 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2219629 T6609 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2219629 T6609 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2219630 T6609 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2219631 T6609 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2219631 T6609 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2219631 T6609 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2219632 T6609 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2219632 T6609 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2219633 T6609 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2219652 T6609 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2219656 T6609 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2219657 T6609 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2219657 T6609 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=50.056640625, floorSegmentMB=0.8681640625, forceMergeDeletesPctAllowed=4.029527315893077, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.85939646723903
   [junit4]   2> 2219658 T6609 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@55787a0b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@390d885),segFN=segments_1,generation=1}
   [junit4]   2> 2219658 T6609 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2219659 T6609 oass.SolrIndexSearcher.<init> Opening Searcher@268b2ccf[collection1] main
   [junit4]   2> 2219659 T6609 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2219660 T6609 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2219660 T6609 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2219661 T6609 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2219661 T6609 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2219661 T6609 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2219662 T6609 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2219662 T6609 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2219662 T6609 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 2219662 T6609 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2219667 T6610 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@268b2ccf[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2219669 T6609 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2219670 T6613 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22675/stu collection:collection1 shard:shard1
   [junit4]   2> 2219671 T6563 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 2219671 T6563 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2219671 T6613 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2219676 T6613 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2219677 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219677 T6613 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2219678 T6613 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1528 name=collection1 org.apache.solr.core.SolrCore@2541c9db url=http://127.0.0.1:22675/stu/collection1 node=127.0.0.1:22675_stu C1528_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:22675/stu, core=collection1, node_name=127.0.0.1:22675_stu}
   [junit4]   2> 2219678 T6613 C1528 P22675 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22675/stu/collection1/
   [junit4]   2> 2219678 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2219678 T6613 C1528 P22675 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2219679 T6613 C1528 P22675 oasc.SyncStrategy.syncToMe http://127.0.0.1:22675/stu/collection1/ has no replicas
   [junit4]   2> 2219679 T6613 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22675/stu/collection1/ shard1
   [junit4]   2> 2219679 T6613 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2219680 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219681 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2219681 T6608 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2219681 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2219684 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2219686 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219788 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2219788 T6608 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2219788 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2219838 T6613 oasc.ZkController.register We are http://127.0.0.1:22675/stu/collection1/ and leader is http://127.0.0.1:22675/stu/collection1/
   [junit4]   2> 2219839 T6613 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22675/stu
   [junit4]   2> 2219839 T6613 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2219839 T6613 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 2219839 T6613 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2219840 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219840 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219841 T6613 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2219841 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219842 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2219842 T6586 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22675/stu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:22675_stu",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 2219844 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2219946 T6608 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2219946 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2219946 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2220702 T6563 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 2220703 T6563 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2220706 T6563 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:53686
   [junit4]   2> 2220706 T6563 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2220706 T6563 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2220707 T6563 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-004
   [junit4]   2> 2220707 T6563 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-004/'
   [junit4]   2> 2220738 T6563 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-004/solr.xml
   [junit4]   2> 2220791 T6563 oasc.CoreContainer.<init> New CoreContainer 48648716
   [junit4]   2> 2220791 T6563 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-004/]
   [junit4]   2> 2220792 T6563 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2220793 T6563 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2220793 T6563 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2220793 T6563 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2220793 T6563 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2220794 T6563 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2220794 T6563 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2220794 T6563 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2220794 T6563 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2220795 T6563 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2220795 T6563 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2220795 T6563 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 2220796 T6563 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:38634/solr
   [junit4]   2> 2220796 T6563 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2220797 T6563 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2220798 T6625 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f149350 name:ZooKeeperConnection Watcher:127.0.0.1:38634 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2220799 T6563 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2220800 T6563 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2220801 T6627 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ef4b515 name:ZooKeeperConnection Watcher:127.0.0.1:38634/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2220801 T6563 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2220806 T6563 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2221810 T6563 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:23175_stu
   [junit4]   2> 2221811 T6563 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:23175_stu
   [junit4]   2> 2221814 T6595 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2221814 T6627 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2221814 T6585 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2221814 T6608 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2221821 T6628 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 2221821 T6628 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2221822 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2221822 T6628 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2221822 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2221823 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2221824 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2221824 T6586 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23175/stu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:23175_stu",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 2221825 T6586 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 2221825 T6586 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 2221826 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2221827 T6627 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2221827 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2221827 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2221827 T6608 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2222823 T6628 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2222824 T6628 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2222824 T6628 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2222825 T6628 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2222825 T6628 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2222826 T6628 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-004/collection1/'
   [junit4]   2> 2222827 T6628 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 2222828 T6628 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 2222828 T6628 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 2222893 T6628 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 2222941 T6628 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2223043 T6628 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2223054 T6628 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2223483 T6628 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2223484 T6628 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2223485 T6628 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2223498 T6628 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2223501 T6628 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2223513 T6628 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2223518 T6628 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2223522 T6628 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2223523 T6628 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2223523 T6628 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2223523 T6628 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2223524 T6628 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2223524 T6628 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2223524 T6628 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2223525 T6628 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2223525 T6628 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-004/collection1/, dataDir=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty2/
   [junit4]   2> 2223525 T6628 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2ca3baae
   [junit4]   2> 2223526 T6628 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty2
   [junit4]   2> 2223526 T6628 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty2/index/
   [junit4]   2> 2223527 T6628 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 2223527 T6628 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-001/jetty2/index
   [junit4]   2> 2223527 T6628 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=41, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=39.0751953125, floorSegmentMB=1.1455078125, forceMergeDeletesPctAllowed=3.3525967927110303, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3165725400804661
   [junit4]   2> 2223528 T6628 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@685ac9d1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@67ce543d),segFN=segments_1,generation=1}
   [junit4]   2> 2223529 T6628 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2223533 T6628 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2223533 T6628 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2223534 T6628 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2223534 T6628 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2223534 T6628 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2223535 T6628 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2223535 T6628 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2223535 T6628 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2223536 T6628 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2223536 T6628 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2223537 T6628 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2223537 T6628 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2223538 T6628 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2223538 T6628 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2223539 T6628 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2223539 T6628 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 2223558 T6628 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2223563 T6628 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2223563 T6628 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2223564 T6628 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=50.056640625, floorSegmentMB=0.8681640625, forceMergeDeletesPctAllowed=4.029527315893077, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.85939646723903
   [junit4]   2> 2223564 T6628 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@685ac9d1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@67ce543d),segFN=segments_1,generation=1}
   [junit4]   2> 2223564 T6628 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2223565 T6628 oass.SolrIndexSearcher.<init> Opening Searcher@34efa807[collection1] main
   [junit4]   2> 2223565 T6628 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2223566 T6628 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2223566 T6628 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2223567 T6628 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2223567 T6628 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2223567 T6628 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2223568 T6628 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2223568 T6628 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2223568 T6628 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 2223569 T6628 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2223572 T6629 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@34efa807[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 2223575 T6628 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2223576 T6632 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:23175/stu collection:collection1 shard:shard2
   [junit4]   2> 2223576 T6563 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 2223577 T6563 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2223577 T6632 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 2223578 T6563 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2223579 T6563 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 30000 for each attempt
   [junit4]   2> 2223579 T6563 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30000
   [junit4]   2> 2223580 T6563 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2223581 T6632 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 2223582 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2223583 T6632 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2223583 T6632 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1529 name=collection1 org.apache.solr.core.SolrCore@735a8083 url=http://127.0.0.1:23175/stu/collection1 node=127.0.0.1:23175_stu C1529_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:23175/stu, core=collection1, node_name=127.0.0.1:23175_stu}
   [junit4]   2> 2223583 T6632 C1529 P23175 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:23175/stu/collection1/
   [junit4]   2> 2223583 T6632 C1529 P23175 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2223584 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2223584 T6632 C1529 P23175 oasc.SyncStrategy.syncToMe http://127.0.0.1:23175/stu/collection1/ has no replicas
   [junit4]   2> 2223584 T6632 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:23175/stu/collection1/ shard2
   [junit4]   2> 2223584 T6632 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 2223586 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2223586 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223586 T6627 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223586 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223586 T6608 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223589 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2223591 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2223692 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223692 T6608 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223692 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223692 T6627 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223743 T6632 oasc.ZkController.register We are http://127.0.0.1:23175/stu/collection1/ and leader is http://127.0.0.1:23175/stu/collection1/
   [junit4]   2> 2223744 T6632 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:23175/stu
   [junit4]   2> 2223744 T6632 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2223744 T6632 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 2223744 T6632 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2223745 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2223746 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2223746 T6632 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2223746 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2223747 T6586 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2223747 T6586 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:23175/stu",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:23175_stu",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 2223749 T6585 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2223851 T6585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223851 T6627 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223851 T6595 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2223851 T6608 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2224583 T6563 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2224584 T6563 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
   [junit4]   2> 2224586 T6593 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=27529,localport=48305], receiveBufferSize:6565888
   [junit4]   2> 2224590 T6593 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=49750,localport=39383], receiveBufferSize=6565888
   [junit4]   2> ASYNC  NEW_CORE C1530 name=collection1 org.apache.solr.core.SolrCore@46934841 url=http://127.0.0.1:48305/stu/collection1 node=127.0.0.1:48305_stu C1530_STATE=coll:control_collection core:collection1 props:{state=active, base_url=http://127.0.0.1:48305/stu, core=collection1, node_name=127.0.0.1:48305_stu, leader=true}
   [junit4]   2> 2224590 T6575 C1530 P48305 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2224591 T6575 C1530 P48305 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2224591 T6575 C1530 P48305 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 2224591 T6575 C1530 P48305 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 2224592 T6575 C1530 P48305 oasup.LogUpdateProcessor.finish [collection1] webapp=/stu path=/update params={softCommit=false&waitSearcher=true&wt=javabin&commit=true&version=2} {commit=} 0 2
   [junit4]   2> 2224694 T6633 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=55418,localport=23175], receiveBufferSize:6565888
   [junit4]   2> 2224695 T6633 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=53686,localport=35855], receiveBufferSize=6565888
   [junit4]   2> 2224697 T6614 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=24547,localport=22675], receiveBufferSize:6565888
   [junit4]   2> 2224698 T6614 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=14016,localport=23105], receiveBufferSize=6565888
   [junit4]   2>  C1528_STATE=coll:collection1 core:collection1 props:{state=active, base_url=http://127.0.0.1:22675/stu, core=collection1, node_name=127.0.0.1:22675_stu, leader=true}
   [junit4]   2> 2224699 T6598 C1528 P22675 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2224699 T6598 C1528 P22675 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 2224700 T6598 C1528 P22675 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 2224700 T6598 C1528 P22675 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 2224700 T6598 C1528 P22675 oasup.LogUpdateProcessor.finish [collection1] webapp=/stu path=/update params={softCommit=false&distrib.from=http://127.0.0.1:23175/stu/collection1/&expungeDeletes=false&commit_end_point=true&waitSearcher=true&wt=javabin&openSearcher=true&update.distrib=FROMLEADER&commit=true&version=2} {commit=} 0 1
   [junit4]   2> 2224702 T6633 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=57888,localport=23175], receiveBufferSize:6565888
   [junit4]   2> 2224702 T6633 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=53686,localport=22599], receiveBufferSize=6565888
   [junit4]   2>  C1529_STATE=coll:collection1 core:collection1 props:{state=active, base_url=http://127.0.0.1:23175/stu, core=collection1, node_name=127.0.0.1:23175_stu, leader=true}
   [junit4]   2> 2224703 T6618 C1529 P23175 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 2224703 T6618 C1529 P23175 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commi

[...truncated too long message...]

Factory - 2 directories currently being tracked
   [junit4]   2> 2882753 T6608 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/c8n_1x3_lf_shard1_replica1/data/index [CachedDir<<refCount=0;path=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/c8n_1x3_lf_shard1_replica1/data/index;done=false>>]
   [junit4]   2> 2882753 T6608 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/c8n_1x3_lf_shard1_replica1/data/index
   [junit4]   2> 2882753 T6608 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/c8n_1x3_lf_shard1_replica1/data [CachedDir<<refCount=0;path=./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/c8n_1x3_lf_shard1_replica1/data;done=false>>]
   [junit4]   2> 2882754 T6608 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001/tempDir-003/c8n_1x3_lf_shard1_replica1/data
   [junit4]   2> 2882754 T6608 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2882754 T6608 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2882754 T6608 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 2882755 T6608 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2882795 T6563 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 23175
   [junit4]   2> 2882796 T6563 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 2882797 T6563 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:38634 38634
   [junit4]   2> 2884055 T6564 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:38634 38634
   [junit4]   2> 2884056 T6563 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:48305/stu
   [junit4]   2> 2884056 T6563 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23175/stu
   [junit4]   2> 2884057 T6563 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:22675/stu
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HttpPartitionTest -Dtests.method=testDistribSearch -Dtests.seed=F3CBDBF4B95B6901 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_SA -Dtests.timezone=SystemV/HST10 -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    672s J1 | HttpPartitionTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F3CBDBF4B95B6901:722D55ECCE04093D]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:545)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:350)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2884075 T6563 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-F3CBDBF4B95B6901-001
   [junit4]   2> NOTE: test params are: codec=Lucene49: {multiDefault=Pulsing41(freqCutoff=1 minBlockSize=82 maxBlockSize=237), range_facet_sl=PostingsFormat(name=NestedPulsing), intDefault=PostingsFormat(name=MockRandom), range_facet_si=Pulsing41(freqCutoff=1 minBlockSize=82 maxBlockSize=237), a_t=Pulsing41(freqCutoff=1 minBlockSize=82 maxBlockSize=237), _version_=PostingsFormat(name=MockRandom), range_facet_l=Pulsing41(freqCutoff=1 minBlockSize=82 maxBlockSize=237), text=PostingsFormat(name=SimpleText), timestamp=Pulsing41(freqCutoff=1 minBlockSize=82 maxBlockSize=237), id=PostingsFormat(name=NestedPulsing)}, docValues:{}, sim=DefaultSimilarity, locale=ar_SA, timezone=SystemV/HST10
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=120637520,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [SuggesterWFSTTest, TestMaxScoreQueryParser, TestAnalyzedSuggestions, HdfsBasicDistributedZkTest, LeaderElectionTest, TestLMJelinekMercerSimilarityFactory, TestExtendedDismaxParser, TestAddFieldRealTimeGet, QueryElevationComponentTest, FileBasedSpellCheckerTest, TestMultiCoreConfBootstrap, SyncSliceTest, TestFiltering, TestUniqueKeyFieldResource, BinaryUpdateRequestHandlerTest, TestClassNameShortening, CacheHeaderTest, JsonLoaderTest, RollingRestartTest, TestSchemaNameResource, OpenExchangeRatesOrgProviderTest, PolyFieldTest, TestCloudManagedSchemaConcurrent, TestSweetSpotSimilarityFactory, BadCopyFieldTest, DocValuesMultiTest, HdfsDirectoryTest, TestSolrXmlPersistor, TestGroupingSearch, TestDocumentBuilder, AnalyticsQueryTest, TestRemoteStreaming, OverseerStatusTest, TestInfoStreamLogging, TestFoldingMultitermQuery, TestManagedResourceStorage, UpdateRequestProcessorFactoryTest, TestFileDictionaryLookup, SliceStateTest, TestSuggestSpellingConverter, CollectionsAPIAsyncDistributedZkTest, ChangedSchemaMergeTest, XsltUpdateRequestHandlerTest, TestSchemaSimilarityResource, CollectionsAPIDistributedZkTest, SolrTestCaseJ4Test, SolrRequestParserTest, CoreAdminHandlerTest, TestMissingGroups, LeaderElectionIntegrationTest, TestRecovery, TestSearcherReuse, BadComponentTest, NotRequiredUniqueKeyTest, TestFieldCollectionResource, DocValuesMissingTest, TestDistributedSearch, SSLMigrationTest, TestWriterPerf, BasicFunctionalityTest, RecoveryZkTest, TestSolr4Spatial2, TestSchemaVersionResource, LoggingHandlerTest, ShardSplitTest, SearchHandlerTest, DistributedSuggestComponentTest, DistributedDebugComponentTest, CoreContainerCoreInitFailuresTest, SolrCoreCheckLockOnStartupTest, UnloadDistributedZkTest, TermsComponentTest, MergeStrategyTest, ZkControllerTest, DistributedQueryComponentOptimizationTest, RAMDirectoryFactoryTest, PrimUtilsTest, TestRestManager, TestPhraseSuggestions, DateMathParserTest, ParsingFieldUpdateProcessorsTest, TestStressLucene, LegacyHTMLStripCharFilterTest, TestPerFieldSimilarity, PathHierarchyTokenizerFactoryTest, ChaosMonkeySafeLeaderTest, DistributedQueryElevationComponentTest, TestSort, TestDocBasedVersionConstraints, DebugComponentTest, ResponseLogComponentTest, SynonymTokenizerTest, BasicDistributedZk2Test, SolrPluginUtilsTest, TestLeaderElectionZkExpiry, TestSchemaResource, TestIndexSearcher, DirectUpdateHandlerTest, TestBM25SimilarityFactory, CachingDirectoryFactoryTest, HttpPartitionTest]
   [junit4] Completed on J1 in 671.96s, 1 test, 1 error <<< FAILURES!

[...truncated 677 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:467: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:923: There were test failures: 407 suites, 1660 tests, 1 error, 51 ignored (17 assumptions)

Total time: 107 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-4.x-Java7 #2025
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 192811967 bytes
Compression is 0.0%
Took 33 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 2029 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/2029/

6 tests failed.
REGRESSION:  org.apache.solr.cloud.CustomCollectionTest.testDistribSearch

Error Message:
expected:<0> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<0> but was:<3>
	at __randomizedtesting.SeedInfo.seed([5B14DAA2FE9BDEDA:DAF254BA89C4BEE6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:248)
	at org.apache.solr.cloud.CustomCollectionTest.doTest(CustomCollectionTest.java:127)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.DistribCursorPagingTest.testDistribSearch

Error Message:
.response.numFound:9!=8

Stack Trace:
junit.framework.AssertionFailedError: .response.numFound:9!=8
	at __randomizedtesting.SeedInfo.seed([5B14DAA2FE9BDEDA:DAF254BA89C4BEE6]:0)
	at junit.framework.Assert.fail(Assert.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:838)
	at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:857)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:560)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:527)
	at org.apache.solr.cloud.DistribCursorPagingTest.doSimpleTest(DistribCursorPagingTest.java:498)
	at org.apache.solr.cloud.DistribCursorPagingTest.doTest(DistribCursorPagingTest.java:89)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.GeneratedMethodAccessor47.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDistribSearch

Error Message:
shard1 is not consistent.  Got 5 from http://127.0.0.1:46862/collection1lastClient and got 556 from http://127.0.0.1:53396/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 5 from http://127.0.0.1:46862/collection1lastClient and got 556 from http://127.0.0.1:53396/collection1
	at __randomizedtesting.SeedInfo.seed([5B14DAA2FE9BDEDA:DAF254BA89C4BEE6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1110)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingDocPerRequestWithHttpSolrServer(FullSolrCloudDistribCmdsTest.java:333)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.doTest(FullSolrCloudDistribCmdsTest.java:134)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.ShardRoutingTest.testDistribSearch

Error Message:
expected:<[f1!f2!doc5/5, f1!f2!doc5, d!doc3, c!doc2, e!doc4]> but was:<[f1!f2!doc5/5, b!doc1, f1!f2!doc5, d!doc3, c!doc2, e!doc4]>

Stack Trace:
java.lang.AssertionError: expected:<[f1!f2!doc5/5, f1!f2!doc5, d!doc3, c!doc2, e!doc4]> but was:<[f1!f2!doc5/5, b!doc1, f1!f2!doc5, d!doc3, c!doc2, e!doc4]>
	at __randomizedtesting.SeedInfo.seed([5B14DAA2FE9BDEDA:DAF254BA89C4BEE6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.doQuery(AbstractFullDistribZkTestBase.java:1414)
	at org.apache.solr.cloud.ShardRoutingTest.doHashingTest(ShardRoutingTest.java:196)
	at org.apache.solr.cloud.ShardRoutingTest.doTest(ShardRoutingTest.java:123)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
shard1 is not consistent.  Got 4 from http://127.0.0.1:64976/collection1lastClient and got 2 from http://127.0.0.1:64979/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 4 from http://127.0.0.1:64976/collection1lastClient and got 2 from http://127.0.0.1:64979/collection1
	at __randomizedtesting.SeedInfo.seed([5B14DAA2FE9BDEDA:DAF254BA89C4BEE6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:145)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node1:{"state":"active","base_url":"http://127.0.0.1:50311/sre","core":"c8n_1x3_lf_shard1_replica1","node_name":"127.0.0.1:50311_sre","leader":"true"}]; clusterState: live nodes:[127.0.0.1:50316_sre, 127.0.0.1:50311_sre] collections:{control_collection=DocCollection(control_collection)={   "router":{"name":"compositeId"},   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{"core_node1":{           "state":"active",           "base_url":"http://127.0.0.1:50311/sre",           "core":"collection1",           "node_name":"127.0.0.1:50311_sre",           "leader":"true"}}}},   "autoCreated":"true",   "replicationFactor":"1",   "maxShardsPerNode":"1"}, collection1=DocCollection(collection1)={   "router":{"name":"compositeId"},   "shards":{     "shard1":{       "range":"80000000-ffffffff",       "state":"active",       "replicas":{"core_node1":{           "state":"active",           "base_url":"http://127.0.0.1:50316/sre",           "core":"collection1",           "node_name":"127.0.0.1:50316_sre",           "leader":"true"}}},     "shard2":{       "range":"0-7fffffff",       "state":"active",       "replicas":{"core_node2":{           "state":"down",           "base_url":"http://127.0.0.1:50320/sre",           "core":"collection1",           "node_name":"127.0.0.1:50320_sre",           "leader":"true"}}}},   "autoCreated":"true",   "replicationFactor":"1",   "maxShardsPerNode":"1"}, c8n_1x2=DocCollection(c8n_1x2)={   "router":{"name":"compositeId"},   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node1":{           "state":"down",           "base_url":"http://127.0.0.1:50320/sre",           "core":"c8n_1x2_shard1_replica1",           "node_name":"127.0.0.1:50320_sre"},         "core_node2":{           "state":"active",           "base_url":"http://127.0.0.1:50311/sre",           "core":"c8n_1x2_shard1_replica2",           "node_name":"127.0.0.1:50311_sre",           "leader":"true"}}}},   "replicationFactor":"2",   "maxShardsPerNode":"1"}, c8n_1x3=DocCollection(c8n_1x3)={   "router":{"name":"compositeId"},   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node1":{           "state":"active",           "base_url":"http://127.0.0.1:50316/sre",           "core":"c8n_1x3_shard1_replica1",           "node_name":"127.0.0.1:50316_sre",           "leader":"true"},         "core_node2":{           "state":"active",           "base_url":"http://127.0.0.1:50311/sre",           "core":"c8n_1x3_shard1_replica2",           "node_name":"127.0.0.1:50311_sre"},         "core_node3":{           "state":"down",           "base_url":"http://127.0.0.1:50320/sre",           "core":"c8n_1x3_shard1_replica3",           "node_name":"127.0.0.1:50320_sre"}}}},   "replicationFactor":"3",   "maxShardsPerNode":"1"}, c8n_1x3_lf=DocCollection(c8n_1x3_lf)={   "router":{"name":"compositeId"},   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node1":{           "state":"active",           "base_url":"http://127.0.0.1:50311/sre",           "core":"c8n_1x3_lf_shard1_replica1",           "node_name":"127.0.0.1:50311_sre",           "leader":"true"},         "core_node2":{           "state":"down",           "base_url":"http://127.0.0.1:50316/sre",           "core":"c8n_1x3_lf_shard1_replica2",           "node_name":"127.0.0.1:50316_sre"},         "core_node3":{           "state":"down",           "base_url":"http://127.0.0.1:50320/sre",           "core":"c8n_1x3_lf_shard1_replica3",           "node_name":"127.0.0.1:50320_sre"}}}},   "replicationFactor":"3",   "maxShardsPerNode":"1"}}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node1:{"state":"active","base_url":"http://127.0.0.1:50311/sre","core":"c8n_1x3_lf_shard1_replica1","node_name":"127.0.0.1:50311_sre","leader":"true"}]; clusterState: live nodes:[127.0.0.1:50316_sre, 127.0.0.1:50311_sre] collections:{control_collection=DocCollection(control_collection)={
  "router":{"name":"compositeId"},
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{"core_node1":{
          "state":"active",
          "base_url":"http://127.0.0.1:50311/sre",
          "core":"collection1",
          "node_name":"127.0.0.1:50311_sre",
          "leader":"true"}}}},
  "autoCreated":"true",
  "replicationFactor":"1",
  "maxShardsPerNode":"1"}, collection1=DocCollection(collection1)={
  "router":{"name":"compositeId"},
  "shards":{
    "shard1":{
      "range":"80000000-ffffffff",
      "state":"active",
      "replicas":{"core_node1":{
          "state":"active",
          "base_url":"http://127.0.0.1:50316/sre",
          "core":"collection1",
          "node_name":"127.0.0.1:50316_sre",
          "leader":"true"}}},
    "shard2":{
      "range":"0-7fffffff",
      "state":"active",
      "replicas":{"core_node2":{
          "state":"down",
          "base_url":"http://127.0.0.1:50320/sre",
          "core":"collection1",
          "node_name":"127.0.0.1:50320_sre",
          "leader":"true"}}}},
  "autoCreated":"true",
  "replicationFactor":"1",
  "maxShardsPerNode":"1"}, c8n_1x2=DocCollection(c8n_1x2)={
  "router":{"name":"compositeId"},
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "state":"down",
          "base_url":"http://127.0.0.1:50320/sre",
          "core":"c8n_1x2_shard1_replica1",
          "node_name":"127.0.0.1:50320_sre"},
        "core_node2":{
          "state":"active",
          "base_url":"http://127.0.0.1:50311/sre",
          "core":"c8n_1x2_shard1_replica2",
          "node_name":"127.0.0.1:50311_sre",
          "leader":"true"}}}},
  "replicationFactor":"2",
  "maxShardsPerNode":"1"}, c8n_1x3=DocCollection(c8n_1x3)={
  "router":{"name":"compositeId"},
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "state":"active",
          "base_url":"http://127.0.0.1:50316/sre",
          "core":"c8n_1x3_shard1_replica1",
          "node_name":"127.0.0.1:50316_sre",
          "leader":"true"},
        "core_node2":{
          "state":"active",
          "base_url":"http://127.0.0.1:50311/sre",
          "core":"c8n_1x3_shard1_replica2",
          "node_name":"127.0.0.1:50311_sre"},
        "core_node3":{
          "state":"down",
          "base_url":"http://127.0.0.1:50320/sre",
          "core":"c8n_1x3_shard1_replica3",
          "node_name":"127.0.0.1:50320_sre"}}}},
  "replicationFactor":"3",
  "maxShardsPerNode":"1"}, c8n_1x3_lf=DocCollection(c8n_1x3_lf)={
  "router":{"name":"compositeId"},
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "state":"active",
          "base_url":"http://127.0.0.1:50311/sre",
          "core":"c8n_1x3_lf_shard1_replica1",
          "node_name":"127.0.0.1:50311_sre",
          "leader":"true"},
        "core_node2":{
          "state":"down",
          "base_url":"http://127.0.0.1:50316/sre",
          "core":"c8n_1x3_lf_shard1_replica2",
          "node_name":"127.0.0.1:50316_sre"},
        "core_node3":{
          "state":"down",
          "base_url":"http://127.0.0.1:50320/sre",
          "core":"c8n_1x3_lf_shard1_replica3",
          "node_name":"127.0.0.1:50320_sre"}}}},
  "replicationFactor":"3",
  "maxShardsPerNode":"1"}}
	at __randomizedtesting.SeedInfo.seed([5B14DAA2FE9BDEDA:DAF254BA89C4BEE6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:368)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11006 lines...]
   [junit4] Suite: org.apache.solr.cloud.SyncSliceTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/init-core-data-001
   [junit4]   2> 312396 T5711 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 312397 T5711 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 312407 T5711 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 312409 T5711 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 312410 T5712 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 312510 T5711 oasc.ZkTestServer.run start zk server on port:64957
   [junit4]   2> 312512 T5711 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 312516 T5718 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2fddcd10 name:ZooKeeperConnection Watcher:127.0.0.1:64957 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 312517 T5711 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 312517 T5711 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 312520 T5711 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 312522 T5720 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44a35be7 name:ZooKeeperConnection Watcher:127.0.0.1:64957/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 312522 T5711 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 312523 T5711 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 312526 T5711 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 312528 T5711 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 312531 T5711 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 312534 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 312534 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 312538 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 312539 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 312642 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 312643 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 312646 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 312647 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 312650 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 312650 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 312653 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 312654 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 312657 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 312658 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 312660 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 312661 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 312664 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 312665 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 312667 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 312668 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 312671 T5711 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 312671 T5711 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 313688 T5711 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 313692 T5711 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:64964
   [junit4]   2> 313693 T5711 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 313693 T5711 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 313693 T5711 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-002
   [junit4]   2> 313694 T5711 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-002/'
   [junit4]   2> 313724 T5711 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-002/solr.xml
   [junit4]   2> 313778 T5711 oasc.CoreContainer.<init> New CoreContainer 1741767645
   [junit4]   2> 313779 T5711 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-002/]
   [junit4]   2> 313780 T5711 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 313780 T5711 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 313780 T5711 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 313781 T5711 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 313781 T5711 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 313781 T5711 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 313781 T5711 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 313782 T5711 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 313782 T5711 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 313783 T5711 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 313783 T5711 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 313783 T5711 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 313784 T5711 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:64957/solr
   [junit4]   2> 313784 T5711 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 313785 T5711 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 313786 T5731 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5bbe976a name:ZooKeeperConnection Watcher:127.0.0.1:64957 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 313787 T5711 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 313789 T5711 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 313790 T5733 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@149e722d name:ZooKeeperConnection Watcher:127.0.0.1:64957/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 313791 T5711 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 313792 T5711 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 313795 T5711 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 313798 T5711 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 313801 T5711 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 313804 T5711 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 313807 T5711 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 313808 T5711 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64964_
   [junit4]   2> 313809 T5711 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:64964_
   [junit4]   2> 313812 T5711 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 313814 T5711 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 313818 T5711 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:64964_
   [junit4]   2> 313818 T5711 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 313820 T5711 oasc.Overseer.start Overseer (id=92091703803510787-127.0.0.1:64964_-n_0000000000) starting
   [junit4]   2> 313823 T5711 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 313831 T5735 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 313832 T5711 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 313834 T5711 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 313836 T5711 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 313838 T5734 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 313843 T5736 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 313844 T5736 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 313844 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 313845 T5736 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 313846 T5734 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 313847 T5734 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64964",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:64964_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 313847 T5734 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 313847 T5734 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 313850 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 313851 T5733 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 314845 T5736 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 314846 T5736 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 314847 T5736 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 314847 T5736 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 314848 T5736 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 314849 T5736 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-002/collection1/'
   [junit4]   2> 314851 T5736 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 314852 T5736 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 314852 T5736 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 314918 T5736 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 314966 T5736 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 315068 T5736 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 315080 T5736 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 315526 T5736 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 315528 T5736 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 315529 T5736 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 315543 T5736 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 315546 T5736 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 315563 T5736 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 315568 T5736 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 315573 T5736 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 315574 T5736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 315574 T5736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 315575 T5736 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 315576 T5736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 315576 T5736 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 315576 T5736 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 315577 T5736 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 315577 T5736 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-002/collection1/, dataDir=./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/control/data/
   [junit4]   2> 315577 T5736 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@14c6f844
   [junit4]   2> 315578 T5736 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/control/data
   [junit4]   2> 315579 T5736 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/control/data/index/
   [junit4]   2> 315579 T5736 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 315579 T5736 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/control/data/index
   [junit4]   2> 315580 T5736 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=76.6435546875, floorSegmentMB=2.05078125, forceMergeDeletesPctAllowed=24.325022537823326, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 315581 T5736 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7e7a868d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@769f2c66),segFN=segments_1,generation=1}
   [junit4]   2> 315581 T5736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 315586 T5736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 315588 T5736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 315588 T5736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 315588 T5736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 315589 T5736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 315589 T5736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 315590 T5736 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 315590 T5736 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 315590 T5736 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 315591 T5736 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 315592 T5736 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 315592 T5736 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 315592 T5736 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 315593 T5736 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 315594 T5736 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 315594 T5736 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 315613 T5736 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 315619 T5736 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 315620 T5736 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 315621 T5736 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=76.4248046875, floorSegmentMB=0.908203125, forceMergeDeletesPctAllowed=3.469995054949868, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 315621 T5736 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7e7a868d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@769f2c66),segFN=segments_1,generation=1}
   [junit4]   2> 315622 T5736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 315622 T5736 oass.SolrIndexSearcher.<init> Opening Searcher@286576c8[collection1] main
   [junit4]   2> 315623 T5736 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 315624 T5736 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 315624 T5736 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 315625 T5736 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 315625 T5736 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 315625 T5736 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 315626 T5736 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 315626 T5736 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 315627 T5736 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 315627 T5736 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 315631 T5737 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@286576c8[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 315634 T5736 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 315635 T5740 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:64964 collection:control_collection shard:shard1
   [junit4]   2> 315635 T5711 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 315636 T5711 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 315636 T5740 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 315637 T5711 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 315639 T5742 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f62d6fc name:ZooKeeperConnection Watcher:127.0.0.1:64957/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 315639 T5711 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 315640 T5711 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 315643 T5740 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 315643 T5711 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 315645 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 315645 T5740 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 315645 T5740 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C121 name=collection1 org.apache.solr.core.SolrCore@7a61f13a url=http://127.0.0.1:64964/collection1 node=127.0.0.1:64964_ C121_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:64964, core=collection1, node_name=127.0.0.1:64964_}
   [junit4]   2> 315646 T5740 C121 P64964 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:64964/collection1/
   [junit4]   2> 315646 T5740 C121 P64964 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 315646 T5734 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 315646 T5740 C121 P64964 oasc.SyncStrategy.syncToMe http://127.0.0.1:64964/collection1/ has no replicas
   [junit4]   2> 315647 T5740 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:64964/collection1/ shard1
   [junit4]   2> 315647 T5740 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 315649 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 315650 T5742 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 315650 T5733 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 315654 T5734 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 315657 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 315760 T5733 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 315760 T5742 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 315806 T5740 oasc.ZkController.register We are http://127.0.0.1:64964/collection1/ and leader is http://127.0.0.1:64964/collection1/
   [junit4]   2> 315807 T5740 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:64964
   [junit4]   2> 315807 T5740 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 315807 T5740 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 315807 T5740 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 315810 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 315810 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 315810 T5740 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 315810 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 315812 T5734 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 315813 T5734 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64964",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:64964_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 315815 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 315917 T5733 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 315918 T5742 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 316831 T5711 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 316831 T5711 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 316834 T5711 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:64976
   [junit4]   2> 316834 T5711 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 316835 T5711 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 316835 T5711 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-003
   [junit4]   2> 316835 T5711 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-003/'
   [junit4]   2> 316866 T5711 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-003/solr.xml
   [junit4]   2> 316919 T5711 oasc.CoreContainer.<init> New CoreContainer 1277647269
   [junit4]   2> 316920 T5711 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-003/]
   [junit4]   2> 316921 T5711 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 316921 T5711 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 316921 T5711 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 316922 T5711 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 316922 T5711 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 316922 T5711 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 316922 T5711 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 316923 T5711 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 316923 T5711 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 316924 T5711 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 316924 T5711 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 316924 T5711 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 316925 T5711 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:64957/solr
   [junit4]   2> 316925 T5711 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 316926 T5711 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316927 T5753 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b6e86b2 name:ZooKeeperConnection Watcher:127.0.0.1:64957 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316928 T5711 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316930 T5711 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316931 T5755 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d1eb4f2 name:ZooKeeperConnection Watcher:127.0.0.1:64957/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316931 T5711 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316937 T5711 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 317940 T5711 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64976_
   [junit4]   2> 317941 T5711 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:64976_
   [junit4]   2> 317944 T5733 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 317944 T5755 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 317944 T5742 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 317953 T5756 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 317953 T5756 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 317954 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 317954 T5756 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 317955 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 317955 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 317956 T5734 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 317957 T5734 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:64976",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:64976_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 317958 T5734 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 317958 T5734 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 317960 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 317961 T5755 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 317961 T5742 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 317961 T5733 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 318955 T5756 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 318956 T5756 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 318957 T5756 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 318957 T5756 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 318958 T5756 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 318958 T5756 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-003/collection1/'
   [junit4]   2> 318960 T5756 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 318961 T5756 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 318961 T5756 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 319026 T5756 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 319075 T5756 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 319177 T5756 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 319189 T5756 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 319633 T5756 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 319634 T5756 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 319635 T5756 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 319650 T5756 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 319653 T5756 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 319670 T5756 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 319675 T5756 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 319679 T5756 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 319680 T5756 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 319681 T5756 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 319681 T5756 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 319682 T5756 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 319682 T5756 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 319683 T5756 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 319683 T5756 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 319683 T5756 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-003/collection1/, dataDir=./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/jetty1/
   [junit4]   2> 319684 T5756 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@14c6f844
   [junit4]   2> 319685 T5756 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/jetty1
   [junit4]   2> 319685 T5756 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/jetty1/index/
   [junit4]   2> 319685 T5756 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 319686 T5756 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-5B14DAA2FE9BDEDA-001/tempDir-001/jetty1/index
   [junit4]   2> 319686 T5756 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=76.6435546875, floorSegmentMB=2.05078125, forceMergeDeletesPctAllowed=24.325022537823326, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 319687 T5756 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@751abffa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@10dacd47),segFN=segments_1,generation=1}
   [junit4]   2> 319688 T5756 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 319695 T5756 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 319695 T5756 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 319696 T5756 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 319696 T5756 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 319696 T5756 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 319697 T5756 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 319697 T5756 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 319698 T5756 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 319698 T5756 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 319699 T5756 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 319699 T5756 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 319700 T5756 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 319700 T5756 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 319701 T5756 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 319702 T5756 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 319702 T5756 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 319721 T5756 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 319726 T5756 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 319727 T5756 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 319727 T5756 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=25, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=76.4248046875, floorSegmentMB=0.908203125, forceMergeDeletesPctAllowed=3.469995054949868, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 319728 T5756 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@751abffa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@10dacd47),segFN=segments_1,generation=1}
   [junit4]   2> 319729 T5756 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 319729 T5756 oass.SolrIndexSearcher.<init> Opening Searcher@3eabe00c[collection1] main
   [junit4]   2> 319729 T5756 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 319731 T5756 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 319731 T5756 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 319732 T5756 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 319732 T5756 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 319732 T5756 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 319733 T5756 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 319733 T5756 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 319734 T5756 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 319734 T5756 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 319739 T5757 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3eabe00c[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 319742 T5756 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 319743 T5760 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:64976 collection:collection1 shard:shard1
   [junit4]   2> 319744 T5711 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 319744 T5711 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 319744 T5760 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 319750 T5760 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 319752 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 319753 T5760 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 319753 T5760 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C122 name=collection1 org.apache.solr.core.SolrCore@12693c90 url=http://127.0.0.1:64976/collection1 node=127.0.0.1:64976_ C122_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:64976, core=collection1, node_name=127.0.0.1:64976_}
   [junit4]   2> 319753 T5760 C122 P64976 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:64976/collection1/
   [junit4]   2> 319754 T5760 C122 P64976 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 319754 T5734 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 319754 T5760 C122 P64976 oasc.SyncStrategy.syncToMe http://127.0.0.1:64976/collection1/ has no replicas
   [junit4]   2> 319754 T5760 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:64976/collection1/ shard1
   [junit4]   2> 319755 T5760 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 319757 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 319758 T5733 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 319758 T5742 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 319758 T5755 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 319762 T5734 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 319764 T5733 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 319867 T5755 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 319867 T5742 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 319867 T5733 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 319913 T5760 oasc.ZkController.register We are http://127.0.0.1

[...truncated too long message...]

ier=3 -Dtests.slow=true -Dtests.locale=fr_LU -Dtests.timezone=Libya -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 66.5s J0 | DistribCursorPagingTest.testDistribSearch <<<
   [junit4]    > Throwable #1: junit.framework.AssertionFailedError: .response.numFound:9!=8
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5B14DAA2FE9BDEDA:DAF254BA89C4BEE6]:0)
   [junit4]    > 	at junit.framework.Assert.fail(Assert.java:50)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:838)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:857)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:560)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:527)
   [junit4]    > 	at org.apache.solr.cloud.DistribCursorPagingTest.doSimpleTest(DistribCursorPagingTest.java:498)
   [junit4]    > 	at org.apache.solr.cloud.DistribCursorPagingTest.doTest(DistribCursorPagingTest.java:89)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 3111106 T5483 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solr.cloud.DistribCursorPagingTest-5B14DAA2FE9BDEDA-001
   [junit4]   2> 66554 T5482 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> 3112197 T5583 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 3112197 T5612 oasc.Overseer$ClusterStateUpdater.checkIfIamStillLeader ERROR could not read the data org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:277)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:274)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:316)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:85)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater$1.run(Overseer.java:288)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Lucene49: {float_last=PostingsFormat(name=TestBloomFilteredLucene41Postings), bcd_str_first=Pulsing41(freqCutoff=3 minBlockSize=33 maxBlockSize=143), legacy_int=PostingsFormat(name=TestBloomFilteredLucene41Postings), legacy_sortable_float_first=MockVariableIntBlock(baseBlockSize=25), str_first=Pulsing41(freqCutoff=3 minBlockSize=33 maxBlockSize=143), float=Pulsing41(freqCutoff=3 minBlockSize=33 maxBlockSize=143), collation_en_primary_first=PostingsFormat(name=MockSep), bcd_str=PostingsFormat(name=TestBloomFilteredLucene41Postings), icu_collation_en_primary_first=PostingsFormat(name=MockSep), legacy_sortable_int_last=PostingsFormat(name=MockSep), bcd_int=PostingsFormat(name=MockSep), id=PostingsFormat(name=MockSep), legacy_sortable_float_last=MockVariableIntBlock(baseBlockSize=25), legacy_sortable_int_first=PostingsFormat(name=MockSep), int=PostingsFormat(name=MockSep), legacy_float=MockVariableIntBlock(baseBlockSize=25), float_first=PostingsFormat(name=TestBloomFilteredLucene41Postings), str_last=Pulsing41(freqCutoff=3 minBlockSize=33 maxBlockSize=143), bcd_str_last=Pulsing41(freqCutoff=3 minBlockSize=33 maxBlockSize=143), bcd_int_first=Pulsing41(freqCutoff=3 minBlockSize=33 maxBlockSize=143), legacy_sortable_int=Pulsing41(freqCutoff=3 minBlockSize=33 maxBlockSize=143), str=PostingsFormat(name=TestBloomFilteredLucene41Postings), legacy_sortable_float=PostingsFormat(name=MockSep), collation_en_primary_last=PostingsFormat(name=MockSep), bcd_int_last=MockVariableIntBlock(baseBlockSize=25), int_first=Pulsing41(freqCutoff=3 minBlockSize=33 maxBlockSize=143), int_last=MockVariableIntBlock(baseBlockSize=25), _version_=Pulsing41(freqCutoff=3 minBlockSize=33 maxBlockSize=143), icu_collation_en_primary_last=PostingsFormat(name=MockSep)}, docValues:{str_dv=DocValuesFormat(name=Lucene49), float_dv_last=DocValuesFormat(name=Lucene49), icu_collation_en_primary_dv_last=DocValuesFormat(name=Asserting), str_dv_last=DocValuesFormat(name=SimpleText), int_dv_first=DocValuesFormat(name=Memory), float_dv_first=DocValuesFormat(name=Asserting), float_dv=DocValuesFormat(name=SimpleText), icu_collation_en_primary_dv_first=DocValuesFormat(name=Lucene49), collation_en_primary_dv_first=DocValuesFormat(name=Lucene49), str_dv_first=DocValuesFormat(name=SimpleText), collation_en_primary_dv_last=DocValuesFormat(name=Lucene49), int_dv=DocValuesFormat(name=Asserting), collation_en_primary_dv=DocValuesFormat(name=Memory), icu_collation_en_primary_dv=DocValuesFormat(name=Memory), int_dv_last=DocValuesFormat(name=Memory)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=fr_LU, timezone=Libya
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=246803952,total=535822336
   [junit4]   2> NOTE: All tests run in this JVM: [DirectSolrSpellCheckerTest, ParsingFieldUpdateProcessorsTest, TestJmxIntegration, StatelessScriptUpdateProcessorFactoryTest, TestLazyCores, ScriptEngineTest, TestFiltering, MBeansHandlerTest, TestSchemaVersionResource, IndexBasedSpellCheckerTest, BadCopyFieldTest, SpatialFilterTest, TestLuceneMatchVersion, TestSolr4Spatial, MinimalSchemaTest, TestFastOutputStream, DocValuesTest, TestCloudManagedSchemaConcurrent, CursorMarkTest, TestExceedMaxTermLength, MoreLikeThisHandlerTest, SynonymTokenizerTest, TestRTGBase, TestValueSourceCache, TestSolrXmlPersistence, PeerSyncTest, DeleteShardTest, SSLMigrationTest, TestConfigSets, SuggesterTest, AutoCommitTest, AnalyticsQueryTest, UnloadDistributedZkTest, TestSimpleQParserPlugin, TestXIncludeConfig, OverseerRolesTest, TestUniqueKeyFieldResource, TriLevelCompositeIdRoutingTest, ShowFileRequestHandlerTest, TestFaceting, TestCloudInspectUtil, SolrIndexSplitterTest, TestFieldSortValues, TestUtils, DisMaxRequestHandlerTest, TimeZoneUtilsTest, TestClassNameShortening, AssignTest, TestSolrXmlPersistor, SimplePostToolTest, OverseerCollectionProcessorTest, TestShortCircuitedRequests, TestFieldCollectionResource, TestComponentsName, TestJmxMonitoredMap, FastVectorHighlighterTest, DocumentBuilderTest, TestDistribDocBasedVersion, DistribDocExpirationUpdateProcessorTest, TestCopyFieldCollectionResource, TestSolrQueryParser, SpellCheckComponentTest, TestPHPSerializedResponseWriter, TestRecoveryHdfs, SuggesterFSTTest, TestMultiCoreConfBootstrap, TestWriterPerf, DateFieldTest, CollectionsAPIAsyncDistributedZkTest, TestCoreContainer, TestManagedResourceStorage, ZkSolrClientTest, AtomicUpdatesTest, TestImplicitCoreProperties, InfoHandlerTest, NotRequiredUniqueKeyTest, HdfsSyncSliceTest, TestNoOpRegenerator, QueryEqualityTest, TestIndexSearcher, CSVRequestHandlerTest, LoggingHandlerTest, TestWordDelimiterFilterFactory, HdfsChaosMonkeySafeLeaderTest, DistributedSpellCheckComponentTest, IndexSchemaRuntimeFieldTest, TestIndexingPerformance, TestSuggestSpellingConverter, RankQueryTest, DistributedQueryComponentOptimizationTest, TestSystemIdResolver, TestQueryUtils, TestExtendedDismaxParser, TestPartialUpdateDeduplication, DistributedSuggestComponentTest, TestSurroundQueryParser, TestFastLRUCache, TestIBSimilarityFactory, ReplicationFactorTest, ShardSplitTest, BasicZkTest, TestDistributedSearch, DistribCursorPagingTest]
   [junit4] Completed on J0 in 67.85s, 1 test, 1 failure <<< FAILURES!

[...truncated 589 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:467: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:923: There were test failures: 407 suites, 1660 tests, 6 failures, 51 ignored (17 assumptions)

Total time: 103 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-4.x-Java7 #2025
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 175300752 bytes
Compression is 0.0%
Took 30 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 2028 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/2028/

1 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1
	at __randomizedtesting.SeedInfo.seed([35C846E7509816E4:B42EC8FF27C776D8]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:545)
	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:350)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11348 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/init-core-data-001
   [junit4]   2> 1726025 T12285 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1726031 T12285 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1726032 T12285 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1726032 T12286 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1726132 T12285 oasc.ZkTestServer.run start zk server on port:14532
   [junit4]   2> 1726133 T12285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1726137 T12292 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13acf51c name:ZooKeeperConnection Watcher:127.0.0.1:14532 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1726137 T12285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1726137 T12285 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1726141 T12285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1726142 T12294 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@da83982 name:ZooKeeperConnection Watcher:127.0.0.1:14532/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1726142 T12285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1726142 T12285 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1726145 T12285 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1726146 T12285 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1726148 T12285 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1726150 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1726151 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1726154 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1726154 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1726257 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1726258 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1726260 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1726261 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1726263 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1726263 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1726265 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1726266 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1726268 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1726269 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1726271 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1726272 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1726274 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1726275 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1726277 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1726277 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1726279 T12285 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1726280 T12285 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1727322 T12285 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1727325 T12285 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:65388
   [junit4]   2> 1727325 T12285 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1727326 T12285 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1727326 T12285 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-002
   [junit4]   2> 1727326 T12285 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-002/'
   [junit4]   2> 1727356 T12285 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-002/solr.xml
   [junit4]   2> 1727426 T12285 oasc.CoreContainer.<init> New CoreContainer 1223254099
   [junit4]   2> 1727426 T12285 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-002/]
   [junit4]   2> 1727427 T12285 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1727427 T12285 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1727428 T12285 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1727428 T12285 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1727428 T12285 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1727428 T12285 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1727429 T12285 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1727429 T12285 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1727429 T12285 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1727430 T12285 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1727430 T12285 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1727430 T12285 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1727431 T12285 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14532/solr
   [junit4]   2> 1727431 T12285 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1727432 T12285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1727433 T12305 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@51b30e8a name:ZooKeeperConnection Watcher:127.0.0.1:14532 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1727434 T12285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1727436 T12285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1727437 T12307 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2205a9fb name:ZooKeeperConnection Watcher:127.0.0.1:14532/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1727437 T12285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1727438 T12285 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1727440 T12285 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1727442 T12285 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1727444 T12285 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1727446 T12285 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1727448 T12285 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1727449 T12285 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18198_
   [junit4]   2> 1727450 T12285 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18198_
   [junit4]   2> 1727452 T12285 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1727453 T12285 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1727456 T12285 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:18198_
   [junit4]   2> 1727456 T12285 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1727458 T12285 oasc.Overseer.start Overseer (id=92083445152350211-127.0.0.1:18198_-n_0000000000) starting
   [junit4]   2> 1727460 T12285 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1727466 T12309 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1727466 T12285 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1727467 T12285 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1727468 T12285 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1727470 T12308 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1727475 T12310 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1727475 T12310 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1727476 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727476 T12310 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1727477 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1727478 T12308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18198",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18198_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1727478 T12308 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1727478 T12308 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1727480 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727480 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1728476 T12310 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1728477 T12310 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1728478 T12310 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1728478 T12310 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1728479 T12310 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1728479 T12310 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-002/collection1/'
   [junit4]   2> 1728481 T12310 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1728481 T12310 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1728482 T12310 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1728546 T12310 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1728594 T12310 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1728695 T12310 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1728706 T12310 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1729127 T12310 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1729127 T12310 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1729128 T12310 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1729140 T12310 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1729144 T12310 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1729155 T12310 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1729160 T12310 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1729164 T12310 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1729165 T12310 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1729165 T12310 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1729165 T12310 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1729166 T12310 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1729166 T12310 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1729166 T12310 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1729167 T12310 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1729167 T12310 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-002/collection1/, dataDir=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/control/data/
   [junit4]   2> 1729167 T12310 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1249a33f
   [junit4]   2> 1729168 T12310 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/control/data
   [junit4]   2> 1729168 T12310 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/control/data/index/
   [junit4]   2> 1729169 T12310 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1729169 T12310 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/control/data/index
   [junit4]   2> 1729170 T12310 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=634749715, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1729170 T12310 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f0dcc08 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@317ffd1a),segFN=segments_1,generation=1}
   [junit4]   2> 1729171 T12310 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1729174 T12310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1729175 T12310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1729175 T12310 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1729175 T12310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1729176 T12310 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1729176 T12310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1729176 T12310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1729177 T12310 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1729177 T12310 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1729178 T12310 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1729178 T12310 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1729178 T12310 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1729179 T12310 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1729179 T12310 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1729180 T12310 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1729180 T12310 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1729198 T12310 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1729204 T12310 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1729204 T12310 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1729205 T12310 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=67.21875, floorSegmentMB=0.451171875, forceMergeDeletesPctAllowed=14.992272481182605, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1729206 T12310 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7f0dcc08 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@317ffd1a),segFN=segments_1,generation=1}
   [junit4]   2> 1729206 T12310 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1729206 T12310 oass.SolrIndexSearcher.<init> Opening Searcher@6ebcf5dd[collection1] main
   [junit4]   2> 1729206 T12310 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1729207 T12310 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1729208 T12310 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1729208 T12310 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1729208 T12310 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1729209 T12310 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1729209 T12310 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1729210 T12310 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1729210 T12310 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1729210 T12310 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1729213 T12311 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6ebcf5dd[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1729214 T12310 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1729215 T12314 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:18198 collection:control_collection shard:shard1
   [junit4]   2> 1729215 T12285 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1729216 T12285 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1729216 T12314 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1729220 T12285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1729221 T12317 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b5facfb name:ZooKeeperConnection Watcher:127.0.0.1:14532/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1729221 T12314 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1729221 T12285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1729223 T12285 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1729223 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1729223 T12314 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1729224 T12314 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1053 name=collection1 org.apache.solr.core.SolrCore@1703bd47 url=http://127.0.0.1:18198/collection1 node=127.0.0.1:18198_ C1053_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:18198, core=collection1, node_name=127.0.0.1:18198_}
   [junit4]   2> 1729224 T12314 C1053 P18198 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:18198/collection1/
   [junit4]   2> 1729224 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1729224 T12314 C1053 P18198 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1729225 T12285 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1729225 T12314 C1053 P18198 oasc.SyncStrategy.syncToMe http://127.0.0.1:18198/collection1/ has no replicas
   [junit4]   2> 1729225 T12314 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:18198/collection1/ shard1
   [junit4]   2> 1729225 T12314 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1729226 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1729227 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1729227 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1729230 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1729231 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1729233 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1729334 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1729334 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1729383 T12314 oasc.ZkController.register We are http://127.0.0.1:18198/collection1/ and leader is http://127.0.0.1:18198/collection1/
   [junit4]   2> 1729384 T12314 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:18198
   [junit4]   2> 1729384 T12314 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1729384 T12314 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1729384 T12314 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1729385 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1729386 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1729386 T12314 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1729386 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1729387 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1729387 T12308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18198",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18198_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1729389 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1729491 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1729491 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1730287 T12285 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1730288 T12285 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1730291 T12285 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:18735
   [junit4]   2> 1730291 T12285 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1730291 T12285 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1730292 T12285 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003
   [junit4]   2> 1730292 T12285 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/'
   [junit4]   2> 1730322 T12285 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/solr.xml
   [junit4]   2> 1730391 T12285 oasc.CoreContainer.<init> New CoreContainer 2146848829
   [junit4]   2> 1730391 T12285 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/]
   [junit4]   2> 1730392 T12285 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1730393 T12285 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1730393 T12285 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1730393 T12285 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1730393 T12285 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1730394 T12285 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1730394 T12285 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1730394 T12285 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1730394 T12285 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1730395 T12285 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1730395 T12285 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1730395 T12285 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1730396 T12285 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14532/solr
   [junit4]   2> 1730396 T12285 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1730397 T12285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1730398 T12328 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40e166d name:ZooKeeperConnection Watcher:127.0.0.1:14532 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1730399 T12285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1730401 T12285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1730402 T12330 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10060cd6 name:ZooKeeperConnection Watcher:127.0.0.1:14532/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1730402 T12285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1730406 T12285 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1731409 T12285 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52258_
   [junit4]   2> 1731410 T12285 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52258_
   [junit4]   2> 1731412 T12307 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1731412 T12317 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1731412 T12330 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1731419 T12331 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1731419 T12331 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1731420 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1731420 T12331 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1731420 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1731421 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1731422 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1731422 T12308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52258",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52258_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1731422 T12308 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1731423 T12308 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1731424 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1731425 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1731425 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1731425 T12330 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1732421 T12331 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1732422 T12331 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1732422 T12331 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1732423 T12331 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1732423 T12331 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1732424 T12331 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/collection1/'
   [junit4]   2> 1732425 T12331 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1732426 T12331 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1732426 T12331 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1732490 T12331 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1732538 T12331 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1732639 T12331 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1732650 T12331 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1733069 T12331 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1733071 T12331 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1733071 T12331 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1733084 T12331 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1733087 T12331 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1733099 T12331 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1733103 T12331 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1733107 T12331 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1733108 T12331 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1733109 T12331 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1733109 T12331 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1733110 T12331 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1733110 T12331 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1733110 T12331 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1733110 T12331 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1733111 T12331 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/collection1/, dataDir=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty1/
   [junit4]   2> 1733111 T12331 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1249a33f
   [junit4]   2> 1733112 T12331 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty1
   [junit4]   2> 1733112 T12331 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty1/index/
   [junit4]   2> 1733112 T12331 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1733113 T12331 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty1/index
   [junit4]   2> 1733114 T12331 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=634749715, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1733114 T12331 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@12bbd650 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@18e1153e),segFN=segments_1,generation=1}
   [junit4]   2> 1733115 T12331 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1733119 T12331 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1733120 T12331 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1733120 T12331 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1733120 T12331 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1733121 T12331 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1733121 T12331 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1733121 T12331 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1733122 T12331 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1733122 T12331 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1733123 T12331 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1733123 T12331 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1733123 T12331 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1733124 T12331 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1733124 T12331 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1733125 T12331 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1733125 T12331 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1733144 T12331 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1733148 T12331 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1733149 T12331 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1733149 T12331 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=67.21875, floorSegmentMB=0.451171875, forceMergeDeletesPctAllowed=14.992272481182605, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1733150 T12331 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@12bbd650 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@18e1153e),segFN=segments_1,generation=1}
   [junit4]   2> 1733150 T12331 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1733151 T12331 oass.SolrIndexSearcher.<init> Opening Searcher@36902ee5[collection1] main
   [junit4]   2> 1733151 T12331 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1733152 T12331 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1733152 T12331 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1733153 T12331 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1733153 T12331 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1733153 T12331 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1733154 T12331 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1733154 T12331 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1733154 T12331 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1733154 T12331 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1733159 T12332 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@36902ee5[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1733161 T12331 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1733162 T12335 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:52258 collection:collection1 shard:shard1
   [junit4]   2> 1733162 T12285 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1733163 T12285 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1733163 T12335 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1733168 T12335 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1733169 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1733169 T12335 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1733170 T12335 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1054 name=collection1 org.apache.solr.core.SolrCore@2a690b2e url=http://127.0.0.1:52258/collection1 node=127.0.0.1:52258_ C1054_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:52258, core=collection1, node_name=127.0.0.1:52258_}
   [junit4]   2> 1733170 T12335 C1054 P52258 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:52258/collection1/
   [junit4]   2> 1733170 T12335 C1054 P52258 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1733170 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1733170 T12335 C1054 P52258 oasc.SyncStrategy.syncToMe http://127.0.0.1:52258/collection1/ has no replicas
   [junit4]   2> 1733171 T12335 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:52258/collection1/ shard1
   [junit4]   2> 1733171 T12335 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1733173 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1733173 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1733173 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1733173 T12330 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1733176 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1733178 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1733280 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1733280 T12330 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1733280 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1733328 T12335 oasc.ZkController.register We are http://127.0.0.1:52258/collection1/ and leader is http://127.0.0.1:52258/collection1/
   [junit4]   2> 1733329 T12335 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:52258
   [junit4]   2> 1733329 T12335 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1733329 T12335 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1733329 T12335 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1733331 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1733331 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1733331 T12335 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1733331 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1733332 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1733333 T12308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52258",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52258_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1733335 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1733436 T12330 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1733436 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1733436 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1734249 T12285 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1734250 T12285 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1734253 T12285 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:13698
   [junit4]   2> 1734254 T12285 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1734254 T12285 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1734254 T12285 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-004
   [junit4]   2> 1734254 T12285 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-004/'
   [junit4]   2> 1734284 T12285 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-004/solr.xml
   [junit4]   2> 1734353 T12285 oasc.CoreContainer.<init> New CoreContainer 546603471
   [junit4]   2> 1734354 T12285 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-004/]
   [junit4]   2> 1734355 T12285 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1734355 T12285 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1734355 T12285 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1734355 T12285 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1734356 T12285 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1734356 T12285 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1734356 T12285 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1734356 T12285 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1734357 T12285 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1734357 T12285 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1734357 T12285 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1734358 T12285 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1734358 T12285 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:14532/solr
   [junit4]   2> 1734358 T12285 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1734359 T12285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1734361 T12347 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75d64dbe name:ZooKeeperConnection Watcher:127.0.0.1:14532 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1734362 T12285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1734363 T12285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1734364 T12349 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ec21750 name:ZooKeeperConnection Watcher:127.0.0.1:14532/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1734365 T12285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1734370 T12285 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1735373 T12285 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62399_
   [junit4]   2> 1735374 T12285 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62399_
   [junit4]   2> 1735376 T12349 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1735376 T12307 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1735376 T12330 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1735376 T12317 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1735383 T12350 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1735383 T12350 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1735384 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1735384 T12350 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1735384 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1735385 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1735385 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1735386 T12308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62399",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:62399_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1735386 T12308 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1735387 T12308 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1735388 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1735389 T12330 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1735389 T12349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1735389 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1735389 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1736385 T12350 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1736386 T12350 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1736386 T12350 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1736387 T12350 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1736387 T12350 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1736388 T12350 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-004/collection1/'
   [junit4]   2> 1736389 T12350 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1736390 T12350 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1736390 T12350 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1736455 T12350 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1736504 T12350 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1736605 T12350 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1736616 T12350 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1737037 T12350 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1737038 T12350 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1737039 T12350 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1737051 T12350 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1737054 T12350 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1737066 T12350 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1737071 T12350 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1737075 T12350 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1737076 T12350 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1737076 T12350 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1737076 T12350 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1737077 T12350 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1737077 T12350 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1737077 T12350 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1737078 T12350 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1737078 T12350 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-004/collection1/, dataDir=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty2/
   [junit4]   2> 1737078 T12350 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1249a33f
   [junit4]   2> 1737079 T12350 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty2
   [junit4]   2> 1737079 T12350 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty2/index/
   [junit4]   2> 1737080 T12350 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1737080 T12350 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-001/jetty2/index
   [junit4]   2> 1737081 T12350 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=634749715, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1737082 T12350 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53f5b9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37c20696),segFN=segments_1,generation=1}
   [junit4]   2> 1737082 T12350 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1737087 T12350 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1737087 T12350 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1737087 T12350 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1737088 T12350 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1737088 T12350 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1737088 T12350 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1737089 T12350 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1737089 T12350 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1737089 T12350 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1737090 T12350 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1737090 T12350 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1737091 T12350 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1737091 T12350 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1737092 T12350 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1737092 T12350 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1737093 T12350 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1737111 T12350 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1737115 T12350 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1737116 T12350 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1737116 T12350 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=22, maxMergedSegmentMB=67.21875, floorSegmentMB=0.451171875, forceMergeDeletesPctAllowed=14.992272481182605, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1737117 T12350 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@53f5b9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@37c20696),segFN=segments_1,generation=1}
   [junit4]   2> 1737117 T12350 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1737118 T12350 oass.SolrIndexSearcher.<init> Opening Searcher@73a4684a[collection1] main
   [junit4]   2> 1737118 T12350 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1737119 T12350 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1737119 T12350 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1737119 T12350 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1737120 T12350 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1737120 T12350 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1737121 T12350 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1737121 T12350 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1737121 T12350 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1737121 T12350 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1737125 T12351 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@73a4684a[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1737128 T12350 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1737128 T12354 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62399 collection:collection1 shard:shard2
   [junit4]   2> 1737129 T12285 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1737129 T12285 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1737129 T12354 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1737131 T12285 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1737131 T12285 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 30000 for each attempt
   [junit4]   2> 1737132 T12285 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30000
   [junit4]   2> 1737132 T12285 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1737134 T12354 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1737135 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1737135 T12354 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1737136 T12354 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1055 name=collection1 org.apache.solr.core.SolrCore@1c235676 url=http://127.0.0.1:62399/collection1 node=127.0.0.1:62399_ C1055_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:62399, core=collection1, node_name=127.0.0.1:62399_}
   [junit4]   2> 1737136 T12354 C1055 P62399 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62399/collection1/
   [junit4]   2> 1737136 T12354 C1055 P62399 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1737136 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1737136 T12354 C1055 P62399 oasc.SyncStrategy.syncToMe http://127.0.0.1:62399/collection1/ has no replicas
   [junit4]   2> 1737137 T12354 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62399/collection1/ shard2
   [junit4]   2> 1737137 T12354 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1737139 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1737139 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737139 T12330 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737139 T12349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737139 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737142 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1737144 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1737245 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737245 T12349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737245 T12330 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737245 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737294 T12354 oasc.ZkController.register We are http://127.0.0.1:62399/collection1/ and leader is http://127.0.0.1:62399/collection1/
   [junit4]   2> 1737295 T12354 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62399
   [junit4]   2> 1737295 T12354 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1737295 T12354 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1737295 T12354 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1737296 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1737297 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1737297 T12354 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1737297 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1737298 T12308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1737298 T12308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62399",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:62399_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1737300 T12307 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1737402 T12317 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737402 T12349 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737402 T12307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1737402 T12330 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1738134 T12285 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1738135 T12285 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
   [junit4]   2> 1738136 T12315 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=16320,localport=18198], receiveBufferSize:6565888
   [junit4]   2> 1738140 T12315 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=65388,localport=58267], receiveBufferSize=6565888
   [junit4]   2> ASYNC  NEW_CORE C1056 name=collection1 org.apache.solr.core.SolrCore@1703bd47 url=http://127.0.0.1:18198/collection1 node=127.0.0.1:18198_ C1056_STATE=coll:control_collection core:collection1 props:{state=active, base_url=http://127.0.0.1:18198, core=collection1, node_name=127.0.0.1:18198_, leader=true}
   [junit4]   2> 1738140 T12297 C1056 P18198 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1738142 T12297 C1056 P18198 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1738142 T12297 C1056 P18198 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 1738143 T12297 C1056 P18198 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 1738143 T12297 C1056 P18198 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&softCommit=false&commit=true&wt=javabin&version=2} {commit=} 0 3
   [junit4]   2> 1738245 T12336 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=30931,localport=52258], receiveBufferSize:6565888
   [junit4]   2> 1738246 T12336 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=18735,localport=65173], receiveBufferSize=6565888
   [junit4]   2> 1738247 T12336 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=59423,localport=52258], receiveBufferSize:6565888
   [junit4]   2> 1738248 T12336 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=18735,localport=25557], receiveBufferSize=6565888
   [junit4]   2> ASYNC  NEW_CORE C1057 name=collection1 org.apache.solr.core.SolrCore@2a690b2e url=http://127.0.0.1:52258/collection1 node=127.0.0.1:52258_ C1057_STATE=coll:collection1 core:collection1 props:{state=active, base_url=http://127.0.0.1:52258, core=collection1, node_name=127.0.0.1:52258_, leader=true}
   [junit4]   2> 1738248 T12321 C1057 P52258 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1738249 T12321 C1057 P52258 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 1738250 T12321 C1057 P52258 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIndexSearcher
   [junit4]   2> 1738250 T12321 C1057 P52258 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 1738250 T12321 C1057 P52258 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&softCommit=false&openSearcher=true&distrib.from=http://127.0.0.1:52258/collection1/&update.distrib=FROMLEADER&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2} {commit=} 0 2
   [junit4]   2> 1738252 T12355 oasc.SocketProxy$Acceptor.run accepted Socket[addr=/140.211.11.196,port=19975,localport=62399], receiveBufferSize:6565888
   [junit4]   2> 1738253 T12355 oasc.SocketProxy$Bridge.<init> proxy connection Socket[addr=/127.0.0.1,port=13698,localport=55024], receiveBufferSize=6565888
   [junit4]   2>  C1055_STATE=coll:collection1 core:collection1 props:{state=active, base_ur

[...truncated too long message...]

ory - 2 directories currently being tracked
   [junit4]   2> 2298447 T12330 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/c8n_1x2_shard1_replica1/data/index.20140711153413317 [CachedDir<<refCount=0;path=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/c8n_1x2_shard1_replica1/data/index.20140711153413317;done=false>>]
   [junit4]   2> 2298447 T12330 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/c8n_1x2_shard1_replica1/data/index.20140711153413317
   [junit4]   2> 2298447 T12330 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/c8n_1x2_shard1_replica1/data [CachedDir<<refCount=0;path=./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/c8n_1x2_shard1_replica1/data;done=false>>]
   [junit4]   2> 2298447 T12330 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.HttpPartitionTest-35C846E7509816E4-001/tempDir-003/c8n_1x2_shard1_replica1/data
   [junit4]   2> 2298448 T12330 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/c8n_1x3_lf/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 2298448 T12330 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2298449 T12330 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2298449 T12330 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2298449 T12330 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HttpPartitionTest -Dtests.method=testDistribSearch -Dtests.seed=35C846E7509816E4 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ru_RU -Dtests.timezone=Europe/Kiev -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    572s J1 | HttpPartitionTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([35C846E7509816E4:B42EC8FF27C776D8]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:545)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:350)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2298470 T12285 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-35C846E7509816E4-001
   [junit4]   2> NOTE: test params are: codec=Lucene41, sim=DefaultSimilarity, locale=ru_RU, timezone=Europe/Kiev
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=267880448,total=520093696
   [junit4]   2> NOTE: All tests run in this JVM: [DocumentAnalysisRequestHandlerTest, ScriptEngineTest, TestIBSimilarityFactory, AnalyticsQueryTest, TestUtils, TestCollationFieldDocValues, TestHighlightDedupGrouping, ReturnFieldsTest, TestSolr4Spatial2, SolrCoreCheckLockOnStartupTest, HdfsLockFactoryTest, TestSolrXml, TestMultiCoreConfBootstrap, PolyFieldTest, BasicDistributedZkTest, DocValuesMultiTest, TestManagedSchemaFieldResource, TestFuzzyAnalyzedSuggestions, EchoParamsTest, SuggesterFSTTest, UnloadDistributedZkTest, TestDynamicFieldCollectionResource, TestJmxIntegration, UUIDFieldTest, SystemInfoHandlerTest, AliasIntegrationTest, TestSolrQueryParser, TestRequestStatusCollectionAPI, RollingRestartTest, TestQueryTypes, TermVectorComponentTest, TestBinaryResponseWriter, SolrRequestParserTest, SolrInfoMBeanTest, TestFastOutputStream, DocumentBuilderTest, HdfsChaosMonkeySafeLeaderTest, QueryResultKeyTest, TestFileDictionaryLookup, TestRecoveryHdfs, TestCSVResponseWriter, DirectSolrConnectionTest, QueryEqualityTest, DirectUpdateHandlerOptimizeTest, TestOverriddenPrefixQueryForCustomFieldType, CurrencyFieldOpenExchangeTest, TestValueSourceCache, TestExtendedDismaxParser, ShardRoutingTest, TestNRTOpen, TestComponentsName, TestManagedSchema, HdfsBasicDistributedZk2Test, OverseerTest, TestLMDirichletSimilarityFactory, ParsingFieldUpdateProcessorsTest, TestSchemaVersionResource, FieldAnalysisRequestHandlerTest, SpatialFilterTest, LoggingHandlerTest, TestConfigSets, TestSolrDeletionPolicy2, TestFieldCollectionResource, SolrXmlInZkTest, ClusterStateUpdateTest, DistribDocExpirationUpdateProcessorTest, TestFastLRUCache, AssignTest, AddSchemaFieldsUpdateProcessorFactoryTest, HardAutoCommitTest, QueryElevationComponentTest, TestSolr4Spatial, TestNumberUtils, AddBlockUpdateTest, TestRTGBase, OverseerCollectionProcessorTest, SpellingQueryConverterTest, DocValuesTest, TestDistributedGrouping, SyncSliceTest, AlternateDirectoryTest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, TestStressVersions, HdfsRecoveryZkTest, ClusterStateTest, TestLeaderElectionZkExpiry, TestStressLucene, ResponseLogComponentTest, HighlighterMaxOffsetTest, TestStressUserVersions, CurrencyFieldXmlFileTest, CoreAdminCreateDiscoverTest, TestSystemIdResolver, TestReplicationHandlerBackup, TestChildDocTransformer, TestReload, HttpPartitionTest]
   [junit4] Completed on J1 in 572.49s, 1 test, 1 error <<< FAILURES!

[...truncated 722 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:467: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:923: There were test failures: 407 suites, 1660 tests, 1 error, 52 ignored (18 assumptions)

Total time: 104 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-4.x-Java7 #2025
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 185860654 bytes
Compression is 0.0%
Took 31 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 2027 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/2027/

2 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
shard3 is not consistent.  Got 84 from https://127.0.0.1:32199/u_g/collection1lastClient and got 82 from https://127.0.0.1:65244/u_g/collection1

Stack Trace:
java.lang.AssertionError: shard3 is not consistent.  Got 84 from https://127.0.0.1:32199/u_g/collection1lastClient and got 82 from https://127.0.0.1:65244/u_g/collection1
	at __randomizedtesting.SeedInfo.seed([316490DF767A06D4:B0821EC7012566E8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1

Stack Trace:
org.apache.solr.common.SolrException: No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1
	at __randomizedtesting.SeedInfo.seed([316490DF767A06D4:B0821EC7012566E8]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:545)
	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:350)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11140 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/init-core-data-001
   [junit4]   2> 1281567 T2754 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 1281567 T2754 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /u_g/
   [junit4]   2> 1281572 T2754 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1281573 T2754 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1281573 T2755 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1281673 T2754 oasc.ZkTestServer.run start zk server on port:32163
   [junit4]   2> 1281674 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1281676 T2761 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@762755d5 name:ZooKeeperConnection Watcher:127.0.0.1:32163 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1281677 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1281677 T2754 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1281680 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1281681 T2763 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@241b6744 name:ZooKeeperConnection Watcher:127.0.0.1:32163/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1281681 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1281682 T2754 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1281683 T2754 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1281685 T2754 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1281686 T2754 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1281688 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1281689 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1281692 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1281692 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1281796 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1281797 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1281799 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1281800 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1281802 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1281802 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1281804 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1281805 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1281807 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1281808 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1281810 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1281811 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1281813 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1281813 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1281815 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1281816 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1281818 T2754 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1281819 T2754 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1281822 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1281823 T2765 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f2addc8 name:ZooKeeperConnection Watcher:127.0.0.1:32163/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1281823 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1282927 T2754 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1282932 T2754 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1282934 T2754 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:32167
   [junit4]   2> 1282936 T2754 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1282937 T2754 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1282937 T2754 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-002
   [junit4]   2> 1282937 T2754 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-002/'
   [junit4]   2> 1282969 T2754 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-002/solr.xml
   [junit4]   2> 1283022 T2754 oasc.CoreContainer.<init> New CoreContainer 1011592120
   [junit4]   2> 1283023 T2754 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-002/]
   [junit4]   2> 1283024 T2754 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1283024 T2754 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1283024 T2754 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1283024 T2754 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1283025 T2754 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1283025 T2754 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1283025 T2754 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1283025 T2754 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1283026 T2754 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1283029 T2754 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1283029 T2754 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1283030 T2754 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1283030 T2754 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32163/solr
   [junit4]   2> 1283030 T2754 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1283031 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1283032 T2776 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14855a13 name:ZooKeeperConnection Watcher:127.0.0.1:32163 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1283033 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1283034 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1283035 T2778 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a69a53a name:ZooKeeperConnection Watcher:127.0.0.1:32163/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1283035 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1283036 T2754 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1283039 T2754 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1283041 T2754 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1283043 T2754 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1283045 T2754 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1283048 T2754 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1283049 T2754 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32167_u_g
   [junit4]   2> 1283050 T2754 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32167_u_g
   [junit4]   2> 1283052 T2754 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1283053 T2754 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1283056 T2754 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32167_u_g
   [junit4]   2> 1283056 T2754 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1283058 T2754 oasc.Overseer.start Overseer (id=92076157181820932-127.0.0.1:32167_u_g-n_0000000000) starting
   [junit4]   2> 1283060 T2754 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1283067 T2780 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1283067 T2754 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1283069 T2754 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1283070 T2754 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1283072 T2779 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1283076 T2781 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1283076 T2781 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1283077 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1283077 T2781 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1283078 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1283078 T2779 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:32167/u_g",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32167_u_g",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1283078 T2779 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1283079 T2779 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1283080 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1283080 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1284077 T2781 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1284078 T2781 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1284078 T2781 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1284079 T2781 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1284080 T2781 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1284080 T2781 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-002/collection1/'
   [junit4]   2> 1284081 T2781 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1284082 T2781 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1284082 T2781 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1284147 T2781 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1284194 T2781 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1284297 T2781 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1284307 T2781 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1284717 T2781 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1284720 T2781 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1284722 T2781 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1284727 T2781 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1284747 T2781 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1284747 T2781 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1284748 T2781 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-002/collection1/, dataDir=./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/control/data/
   [junit4]   2> 1284748 T2781 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@294205f4
   [junit4]   2> 1284750 T2781 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/control/data
   [junit4]   2> 1284751 T2781 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/control/data/index/
   [junit4]   2> 1284752 T2781 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1284754 T2781 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/control/data/index
   [junit4]   2> 1284754 T2781 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=9.7978515625, floorSegmentMB=1.671875, forceMergeDeletesPctAllowed=2.5281512159435984, segmentsPerTier=20.0, maxCFSSegmentSizeMB=1.998046875, noCFSRatio=1.0
   [junit4]   2> 1284778 T2781 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1284779 T2781 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1284784 T2781 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1284784 T2781 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1284784 T2781 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1284785 T2781 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1284785 T2781 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1284785 T2781 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1284786 T2781 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1284786 T2781 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1284786 T2781 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1284787 T2781 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1284787 T2781 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1284788 T2781 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1284788 T2781 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1284789 T2781 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1284789 T2781 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1284790 T2781 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1284809 T2781 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1284812 T2781 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1284812 T2781 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1284813 T2781 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=20, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1284815 T2781 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 1284815 T2781 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1284816 T2781 oass.SolrIndexSearcher.<init> Opening Searcher@77773e23[collection1] main
   [junit4]   2> 1284816 T2781 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1284817 T2781 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1284817 T2781 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1284818 T2781 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1284818 T2781 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1284818 T2781 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1284819 T2781 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1284819 T2781 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1284819 T2781 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1284820 T2781 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1284823 T2782 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@77773e23[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1284825 T2781 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1284826 T2785 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:32167/u_g collection:control_collection shard:shard1
   [junit4]   2> 1284826 T2754 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1284827 T2754 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1284827 T2785 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1284832 T2785 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1284832 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1284834 T2787 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6abab3cc name:ZooKeeperConnection Watcher:127.0.0.1:32163/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1284834 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1284834 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1284834 T2785 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1284835 T2785 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C788 name=collection1 org.apache.solr.core.SolrCore@25684adc url=https://127.0.0.1:32167/u_g/collection1 node=127.0.0.1:32167_u_g C788_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:32167/u_g, core=collection1, node_name=127.0.0.1:32167_u_g}
   [junit4]   2> 1284835 T2785 C788 P32167 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:32167/u_g/collection1/
   [junit4]   2> 1284835 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1284835 T2754 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1284835 T2785 C788 P32167 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1284836 T2785 C788 P32167 oasc.SyncStrategy.syncToMe https://127.0.0.1:32167/u_g/collection1/ has no replicas
   [junit4]   2> 1284837 T2785 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:32167/u_g/collection1/ shard1
   [junit4]   2> 1284837 T2785 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1284838 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1284838 T2754 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1284838 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1284838 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1284841 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1284843 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1284845 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1284946 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1284946 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1284994 T2785 oasc.ZkController.register We are https://127.0.0.1:32167/u_g/collection1/ and leader is https://127.0.0.1:32167/u_g/collection1/
   [junit4]   2> 1284995 T2785 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:32167/u_g
   [junit4]   2> 1284995 T2785 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1284995 T2785 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1284996 T2785 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1285006 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1285006 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1285007 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1285007 T2785 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1285008 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1285009 T2779 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:32167/u_g",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32167_u_g",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1285011 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1285114 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1285114 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1285972 T2754 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1285973 T2754 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1285977 T2754 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1285979 T2754 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:32171
   [junit4]   2> 1285981 T2754 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1285981 T2754 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1285981 T2754 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-003
   [junit4]   2> 1285981 T2754 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-003/'
   [junit4]   2> 1286013 T2754 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-003/solr.xml
   [junit4]   2> 1286066 T2754 oasc.CoreContainer.<init> New CoreContainer 1100798904
   [junit4]   2> 1286067 T2754 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-003/]
   [junit4]   2> 1286068 T2754 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1286068 T2754 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1286068 T2754 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1286068 T2754 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1286069 T2754 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1286069 T2754 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1286069 T2754 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1286069 T2754 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1286070 T2754 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1286073 T2754 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1286073 T2754 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1286074 T2754 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1286074 T2754 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32163/solr
   [junit4]   2> 1286074 T2754 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1286075 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1286076 T2798 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e4ec47 name:ZooKeeperConnection Watcher:127.0.0.1:32163 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1286077 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1286079 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1286079 T2800 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3055145d name:ZooKeeperConnection Watcher:127.0.0.1:32163/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1286080 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1286084 T2754 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1287087 T2754 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32171_u_g
   [junit4]   2> 1287088 T2754 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32171_u_g
   [junit4]   2> 1287091 T2800 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1287091 T2787 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1287091 T2778 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1287098 T2801 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1287098 T2801 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1287099 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1287099 T2801 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1287099 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1287100 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1287101 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1287101 T2779 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:32171/u_g",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32171_u_g",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1287102 T2779 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 1287102 T2779 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 1287104 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1287104 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1287104 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1287104 T2800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1288100 T2801 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1288101 T2801 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1288101 T2801 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1288102 T2801 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1288102 T2801 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1288103 T2801 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-003/collection1/'
   [junit4]   2> 1288104 T2801 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1288105 T2801 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1288105 T2801 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1288170 T2801 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1288218 T2801 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1288320 T2801 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1288331 T2801 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1288738 T2801 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1288741 T2801 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1288743 T2801 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1288748 T2801 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1288768 T2801 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1288768 T2801 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1288768 T2801 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-003/collection1/, dataDir=./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty1/
   [junit4]   2> 1288769 T2801 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@294205f4
   [junit4]   2> 1288770 T2801 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty1
   [junit4]   2> 1288771 T2801 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty1/index/
   [junit4]   2> 1288772 T2801 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1288774 T2801 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty1/index
   [junit4]   2> 1288775 T2801 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=9.7978515625, floorSegmentMB=1.671875, forceMergeDeletesPctAllowed=2.5281512159435984, segmentsPerTier=20.0, maxCFSSegmentSizeMB=1.998046875, noCFSRatio=1.0
   [junit4]   2> 1288780 T2801 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 1288780 T2801 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1288786 T2801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1288786 T2801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1288787 T2801 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1288787 T2801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1288787 T2801 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1288788 T2801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1288788 T2801 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1288789 T2801 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1288789 T2801 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1288790 T2801 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1288790 T2801 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1288790 T2801 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1288791 T2801 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1288791 T2801 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1288792 T2801 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1288793 T2801 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1288811 T2801 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1288814 T2801 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1288815 T2801 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1288816 T2801 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=20, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1288818 T2801 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 1288818 T2801 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1288818 T2801 oass.SolrIndexSearcher.<init> Opening Searcher@4a3cc66a[collection1] main
   [junit4]   2> 1288819 T2801 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1288820 T2801 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1288820 T2801 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1288820 T2801 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1288821 T2801 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1288821 T2801 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1288821 T2801 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1288822 T2801 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1288822 T2801 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1288822 T2801 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1288826 T2802 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4a3cc66a[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1288829 T2801 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1288830 T2805 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:32171/u_g collection:collection1 shard:shard3
   [junit4]   2> 1288830 T2754 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1288831 T2754 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1288831 T2805 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 1288836 T2805 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 1288837 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1288837 T2805 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1288838 T2805 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C789 name=collection1 org.apache.solr.core.SolrCore@64c070df url=https://127.0.0.1:32171/u_g/collection1 node=127.0.0.1:32171_u_g C789_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:32171/u_g, core=collection1, node_name=127.0.0.1:32171_u_g}
   [junit4]   2> 1288838 T2805 C789 P32171 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:32171/u_g/collection1/
   [junit4]   2> 1288838 T2805 C789 P32171 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1288838 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1288838 T2805 C789 P32171 oasc.SyncStrategy.syncToMe https://127.0.0.1:32171/u_g/collection1/ has no replicas
   [junit4]   2> 1288839 T2805 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:32171/u_g/collection1/ shard3
   [junit4]   2> 1288839 T2805 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 1288841 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1288841 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1288841 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1288841 T2800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1288844 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1288846 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1288948 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1288948 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1288948 T2800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1288996 T2805 oasc.ZkController.register We are https://127.0.0.1:32171/u_g/collection1/ and leader is https://127.0.0.1:32171/u_g/collection1/
   [junit4]   2> 1288997 T2805 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:32171/u_g
   [junit4]   2> 1288997 T2805 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1288997 T2805 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1288997 T2805 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1288998 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1288999 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1288999 T2805 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1288999 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1289000 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1289001 T2779 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:32171/u_g",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32171_u_g",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1289003 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1289104 T2800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1289104 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1289104 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1289961 T2754 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1289962 T2754 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1289966 T2754 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1289967 T2754 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:32174
   [junit4]   2> 1289970 T2754 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1289970 T2754 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1289970 T2754 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-004
   [junit4]   2> 1289970 T2754 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-004/'
   [junit4]   2> 1290011 T2754 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-004/solr.xml
   [junit4]   2> 1290067 T2754 oasc.CoreContainer.<init> New CoreContainer 581669127
   [junit4]   2> 1290067 T2754 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-004/]
   [junit4]   2> 1290068 T2754 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1290069 T2754 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1290069 T2754 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1290069 T2754 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1290069 T2754 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1290070 T2754 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1290070 T2754 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1290070 T2754 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1290070 T2754 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1290074 T2754 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1290074 T2754 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1290075 T2754 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1290075 T2754 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32163/solr
   [junit4]   2> 1290075 T2754 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1290076 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1290078 T2816 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@709b42fb name:ZooKeeperConnection Watcher:127.0.0.1:32163 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1290078 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1290080 T2754 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1290081 T2818 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2cadde07 name:ZooKeeperConnection Watcher:127.0.0.1:32163/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1290081 T2754 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1290086 T2754 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1291089 T2754 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32174_u_g
   [junit4]   2> 1291090 T2754 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32174_u_g
   [junit4]   2> 1291093 T2778 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1291093 T2800 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1291093 T2818 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1291093 T2787 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1291101 T2819 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1291101 T2819 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1291102 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1291102 T2819 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1291102 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1291103 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1291103 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1291104 T2779 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:32174/u_g",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32174_u_g",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1291104 T2779 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 1291104 T2779 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1291106 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1291107 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1291107 T2800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1291107 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1291107 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1292104 T2819 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1292105 T2819 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1292105 T2819 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1292106 T2819 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1292106 T2819 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1292107 T2819 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-004/collection1/'
   [junit4]   2> 1292108 T2819 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1292109 T2819 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1292109 T2819 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1292175 T2819 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_10
   [junit4]   2> 1292223 T2819 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1292325 T2819 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1292336 T2819 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1292726 T2819 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1292729 T2819 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1292731 T2819 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1292735 T2819 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1292777 T2819 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1292778 T2819 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 1292778 T2819 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-004/collection1/, dataDir=./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty2/
   [junit4]   2> 1292778 T2819 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@294205f4
   [junit4]   2> 1292780 T2819 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty2
   [junit4]   2> 1292781 T2819 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty2/index/
   [junit4]   2> 1292782 T2819 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1292783 T2819 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty2/index
   [junit4]   2> 1292784 T2819 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=9.7978515625, floorSegmentMB=1.671875, forceMergeDeletesPctAllowed=2.5281512159435984, segmentsPerTier=20.0, maxCFSSegmentSizeMB=1.998046875, noCFSRatio=1.0
   [junit4]   2> 1292789 T2819 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 1292789 T2819 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1292795 T2819 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1292795 T2819 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1292795 T2819 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1292796 T2819 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1292796 T2819 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1292796 T2819 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1292797 T2819 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1292797 T2819 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1292797 T2819 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1292798 T2819 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1292799 T2819 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1292799 T2819 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1292799 T2819 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1292800 T2819 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1292800 T2819 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1292801 T2819 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1292820 T2819 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1292823 T2819 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1292823 T2819 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1292824 T2819 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=20, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1292826 T2819 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-001/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 1292827 T2819 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1292827 T2819 oass.SolrIndexSearcher.<init> Opening Searcher@2ebedacd[collection1] main
   [junit4]   2> 1292827 T2819 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1292828 T2819 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1292828 T2819 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1292829 T2819 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1292829 T2819 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1292829 T2819 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1292830 T2819 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1292830 T2819 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1292831 T2819 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1292831 T2819 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1292835 T2820 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2ebedacd[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1292838 T2819 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1292839 T2823 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:32174/u_g collection:collection1 shard:shard2
   [junit4]   2> 1292839 T2754 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1292839 T2754 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1292840 T2823 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1292844 T2823 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1292846 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1292846 T2823 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1292846 T2823 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C790 name=collection1 org.apache.solr.core.SolrCore@437ffc8d url=https://127.0.0.1:32174/u_g/collection1 node=127.0.0.1:32174_u_g C790_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:32174/u_g, core=collection1, node_name=127.0.0.1:32174_u_g}
   [junit4]   2> 1292846 T2823 C790 P32174 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:32174/u_g/collection1/
   [junit4]   2> 1292847 T2823 C790 P32174 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1292847 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1292847 T2823 C790 P32174 oasc.SyncStrategy.syncToMe https://127.0.0.1:32174/u_g/collection1/ has no replicas
   [junit4]   2> 1292847 T2823 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:32174/u_g/collection1/ shard2
   [junit4]   2> 1292848 T2823 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1292849 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1292850 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1292850 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1292850 T2800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1292850 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1292853 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1292855 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1292957 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1292957 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1292957 T2800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1292957 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1293004 T2823 oasc.ZkController.register We are https://127.0.0.1:32174/u_g/collection1/ and leader is https://127.0.0.1:32174/u_g/collection1/
   [junit4]   2> 1293005 T2823 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:32174/u_g
   [junit4]   2> 1293005 T2823 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1293005 T2823 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1293005 T2823 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1293006 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293007 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293007 T2823 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1293007 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293008 T2779 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1293009 T2779 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:32174/u_g",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:32174_u_g",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1293011 T2778 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1293113 T2818 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1293113 T2800 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1293113 T2778 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1293113 T2787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1293968 T2754 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 1293969 T2754 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1293974 T2754 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1293975 T2754 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:32178
   [junit4]   2> 1293977 T2754 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1293977 T2754 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1293978 T2754 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.ChaosMonkeySafeLeaderTest-316490DF767A06D4-001/tempDir-005
   [junit4]   2> 1293978 T2754 oasc.SolrResourceLoader.<in

[...truncated too long message...]

DirectoryFactory.close Closing directory: ./solr.cloud.HttpPartitionTest-316490DF767A06D4-001/tempDir-003/c8n_1x3_shard1_replica3/data
   [junit4]   2> 2571046 T9852 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2571046 T9852 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/c8n_1x3_lf/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 2571047 T9852 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 2571047 T9852 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2571048 T9852 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:358)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 2571083 T9807 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 44424
   [junit4]   2> 2571084 T9807 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 2571085 T9807 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:44409 44409
   [junit4]   2> 2571228 T9808 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:44409 44409
   [junit4]   2> 2571229 T9807 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44420/_
   [junit4]   2> 2571229 T9807 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44414/_
   [junit4]   2> 2571230 T9807 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44424/_
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=HttpPartitionTest -Dtests.method=testDistribSearch -Dtests.seed=316490DF767A06D4 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_BH -Dtests.timezone=Etc/GMT-11 -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    571s J1 | HttpPartitionTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.common.SolrException: No registered leader was found after waiting for 60000ms , collection: c8n_1x3_lf slice: shard1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([316490DF767A06D4:B0821EC7012566E8]:0)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkStateReader.getLeaderRetry(ZkStateReader.java:545)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.testRf3WithLeaderFailover(HttpPartitionTest.java:350)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:149)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2571253 T9807 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-316490DF767A06D4-001
   [junit4]   2> 571173 T9806 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene49), sim=DefaultSimilarity, locale=ar_BH, timezone=Etc/GMT-11
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_60 (64-bit)/cpus=16,threads=1,free=164223880,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [TestLFUCache, BasicDistributedZk2Test, HdfsLockFactoryTest, TestDocSet, EchoParamsTest, TestLRUCache, TimeZoneUtilsTest, TestCursorMarkWithoutUniqueKey, ConvertedLegacyTest, DOMUtilTest, TestQueryUtils, OutputWriterTest, SolrCmdDistributorTest, TestDefaultSimilarityFactory, DistributedSpellCheckComponentTest, StatelessScriptUpdateProcessorFactoryTest, TestSearcherReuse, AddBlockUpdateTest, WordBreakSolrSpellCheckerTest, TestDynamicFieldCollectionResource, XsltUpdateRequestHandlerTest, TestOmitPositions, DistributedQueryComponentCustomSortTest, TestFieldTypeResource, TestUpdate, RegexBoostProcessorTest, OverseerCollectionProcessorTest, ShardSplitTest, SignatureUpdateProcessorFactoryTest, SolrCoreTest, SpellCheckComponentTest, BlockDirectoryTest, MergeStrategyTest, TestMultiCoreConfBootstrap, SolrXmlInZkTest, TestFastLRUCache, SuggesterWFSTTest, TestFunctionQuery, TestHashPartitioner, SpellCheckCollatorTest, TestReRankQParserPlugin, TestStressUserVersions, StatsComponentTest, OverseerStatusTest, TestManagedSynonymFilterFactory, DeleteReplicaTest, FileBasedSpellCheckerTest, TestRemoteStreaming, PluginInfoTest, TestPerFieldSimilarity, UpdateRequestProcessorFactoryTest, TestCodecSupport, TestQueryTypes, JsonLoaderTest, TestSearchPerf, FileUtilsTest, FieldAnalysisRequestHandlerTest, DistributedTermsComponentTest, BasicZkTest, TestCloudManagedSchemaConcurrent, TestRTGBase, TestStressVersions, TestMiniSolrCloudCluster, PreAnalyzedUpdateProcessorTest, TestSchemaSimilarityResource, ChaosMonkeyNothingIsSafeTest, DistributedQueryComponentOptimizationTest, TestPartialUpdateDeduplication, TestNRTOpen, AlternateDirectoryTest, ChaosMonkeySafeLeaderTest, TestSerializedLuceneMatchVersion, ParsingFieldUpdateProcessorsTest, TestXIncludeConfig, TestElisionMultitermQuery, DeleteInactiveReplicaTest, TestSimpleQParserPlugin, DistribDocExpirationUpdateProcessorTest, TestManagedResourceStorage, TestSolr4Spatial, HdfsRecoveryZkTest, TestLuceneMatchVersion, TestJmxMonitoredMap, DistributedExpandComponentTest, TestExceedMaxTermLength, FieldMutatingUpdateProcessorTest, DeleteShardTest, TestIBSimilarityFactory, AsyncMigrateRouteKeyTest, SuggesterFSTTest, DefaultValueUpdateProcessorTest, TestFieldResource, TestStandardQParsers, CollectionsAPIDistributedZkTest, TestJoin, UUIDFieldTest, TermsComponentTest, TestDistributedSearch, TestInitQParser, TestStressRecovery, SimplePostToolTest, ClusterStateUpdateTest, HttpPartitionTest]
   [junit4] Completed on J1 in 571.46s, 1 test, 1 error <<< FAILURES!

[...truncated 641 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:467: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:496: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1299: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:923: There were test failures: 407 suites, 1660 tests, 1 error, 1 failure, 51 ignored (17 assumptions)

Total time: 105 minutes 17 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-Tests-4.x-Java7 #2025
Archived 3 artifacts
Archive block size is 32768
Received 0 blocks and 186884191 bytes
Compression is 0.0%
Took 31 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure