You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/05/06 17:08:26 UTC

[JENKINS] Lucene-Solr-4.8-Linux (32bit/jdk1.8.0_20-ea-b11) - Build # 116 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.8-Linux/116/
Java: 32bit/jdk1.8.0_20-ea-b11 -server -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.ShardSplitTest.testDistribSearch

Error Message:
collection already exists: routeFieldColl

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: routeFieldColl
	at __randomizedtesting.SeedInfo.seed([8C030E2FB8362E26:DE58037CF694E1A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:554)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.doRequest(LBHttpSolrServer.java:340)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:301)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:636)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1661)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1615)
	at org.apache.solr.cloud.ShardSplitTest.splitByRouteFieldTest(ShardSplitTest.java:260)
	at org.apache.solr.cloud.ShardSplitTest.doTest(ShardSplitTest.java:102)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10912 lines...]
   [junit4] Suite: org.apache.solr.cloud.ShardSplitTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/init-core-data-001
   [junit4]   2> 132988 T598 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 132993 T598 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 132994 T598 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 132994 T599 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 133094 T598 oasc.ZkTestServer.run start zk server on port:37216
   [junit4]   2> 133095 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133504 T605 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fc136e name:ZooKeeperConnection Watcher:127.0.0.1:37216 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133504 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133504 T598 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 133516 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133518 T607 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1245b98 name:ZooKeeperConnection Watcher:127.0.0.1:37216/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133518 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133519 T598 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 133521 T598 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 133522 T598 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 133524 T598 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 133525 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 133526 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 133529 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 133529 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 133531 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 133532 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 133534 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 133535 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 133546 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 133547 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 133548 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 133549 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 133550 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 133550 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 133552 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 133553 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 133554 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 133555 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 133560 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 133561 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 133563 T598 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 133563 T598 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 133898 T598 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 133918 T598 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:47581
   [junit4]   2> 133921 T598 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 133921 T598 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 133921 T598 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-002
   [junit4]   2> 133922 T598 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-002/'
   [junit4]   2> 133938 T598 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-002/solr.xml
   [junit4]   2> 133981 T598 oasc.CoreContainer.<init> New CoreContainer 19111604
   [junit4]   2> 133981 T598 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-002/]
   [junit4]   2> 133982 T598 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 133982 T598 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 133983 T598 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 133983 T598 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 133983 T598 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 133984 T598 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 133984 T598 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 133985 T598 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 133985 T598 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 133986 T598 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 133986 T598 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 133987 T598 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 133987 T598 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37216/solr
   [junit4]   2> 133988 T598 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 133989 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133991 T621 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1256a1 name:ZooKeeperConnection Watcher:127.0.0.1:37216 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133992 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133994 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133996 T623 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d2077 name:ZooKeeperConnection Watcher:127.0.0.1:37216/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133996 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133998 T598 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 134000 T598 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 134002 T598 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 134004 T598 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 134008 T598 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 134010 T598 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 134011 T598 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47581_
   [junit4]   2> 134012 T598 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:47581_
   [junit4]   2> 134013 T598 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 134015 T598 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 134018 T598 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:47581_
   [junit4]   2> 134018 T598 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 134019 T598 oasc.Overseer.start Overseer (id=91710266293813251-127.0.0.1:47581_-n_0000000000) starting
   [junit4]   2> 134028 T598 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 134037 T625 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 134037 T598 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 134037 T625 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes at 127.0.0.1:47581_
   [junit4]   2> 134039 T598 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 134040 T598 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 134042 T624 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 134043 T626 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 134043 T626 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 134044 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 134044 T626 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 134045 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 134045 T624 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47581",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:47581_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 134045 T624 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 134045 T624 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 134046 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 134047 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 135045 T626 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 135045 T626 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 135045 T626 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 135046 T626 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 135046 T626 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 135046 T626 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-002/collection1/'
   [junit4]   2> 135047 T626 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 135047 T626 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 135048 T626 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 135072 T626 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 135083 T626 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 135084 T626 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 135088 T626 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 135186 T626 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 135187 T626 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 135188 T626 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 135191 T626 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 135205 T626 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 135205 T626 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 135205 T626 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-002/collection1/, dataDir=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/control/data/
   [junit4]   2> 135206 T626 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1a1b028
   [junit4]   2> 135206 T626 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/control/data
   [junit4]   2> 135206 T626 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/control/data/index/
   [junit4]   2> 135206 T626 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 135207 T626 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/control/data/index
   [junit4]   2> 135207 T626 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.504697298399816]
   [junit4]   2> 135208 T626 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@176a5be lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@190ea5c),segFN=segments_1,generation=1}
   [junit4]   2> 135208 T626 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 135210 T626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 135210 T626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 135210 T626 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 135210 T626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 135210 T626 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 135210 T626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 135211 T626 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 135211 T626 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 135211 T626 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 135211 T626 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 135212 T626 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 135212 T626 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 135212 T626 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 135212 T626 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 135212 T626 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 135213 T626 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 135214 T626 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 135215 T626 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 135215 T626 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 135216 T626 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=13, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 135216 T626 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@176a5be lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@190ea5c),segFN=segments_1,generation=1}
   [junit4]   2> 135216 T626 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 135217 T626 oass.SolrIndexSearcher.<init> Opening Searcher@10065c2[collection1] main
   [junit4]   2> 135217 T626 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 135217 T626 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 135218 T626 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 135218 T626 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 135218 T626 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 135218 T626 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 135219 T626 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 135219 T626 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 135219 T626 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 135219 T626 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 135221 T627 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@10065c2[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 135222 T626 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 135222 T630 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:47581 collection:control_collection shard:shard1
   [junit4]   2> 135222 T598 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 135223 T598 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 135223 T630 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 135224 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 135225 T632 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@776d4e name:ZooKeeperConnection Watcher:127.0.0.1:37216/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 135225 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 135226 T598 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 135228 T630 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 135228 T598 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 135229 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 135229 T630 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 135229 T630 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1132 name=collection1 org.apache.solr.core.SolrCore@15b80 url=http://127.0.0.1:47581/collection1 node=127.0.0.1:47581_ C1132_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:47581, core=collection1, node_name=127.0.0.1:47581_}
   [junit4]   2> 135229 T630 C1132 P47581 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:47581/collection1/
   [junit4]   2> 135230 T630 C1132 P47581 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 135230 T630 C1132 P47581 oasc.SyncStrategy.syncToMe http://127.0.0.1:47581/collection1/ has no replicas
   [junit4]   2> 135230 T630 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:47581/collection1/ shard1
   [junit4]   2> 135230 T630 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 135231 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 135235 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 135469 T598 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 135469 T598 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 135471 T598 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:59329
   [junit4]   2> 135472 T598 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 135472 T598 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 135472 T598 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-003
   [junit4]   2> 135473 T598 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-003/'
   [junit4]   2> 135487 T598 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-003/solr.xml
   [junit4]   2> 135506 T598 oasc.CoreContainer.<init> New CoreContainer 929917
   [junit4]   2> 135507 T598 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-003/]
   [junit4]   2> 135507 T598 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 135508 T598 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 135508 T598 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 135509 T598 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 135509 T598 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 135510 T598 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 135510 T598 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 135510 T598 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 135511 T598 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 135512 T598 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 135512 T598 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 135512 T598 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 135513 T598 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37216/solr
   [junit4]   2> 135513 T598 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 135514 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 135586 T643 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e81a5a name:ZooKeeperConnection Watcher:127.0.0.1:37216 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 135586 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 135589 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 135589 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 135590 T632 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 135590 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 135591 T645 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a75754 name:ZooKeeperConnection Watcher:127.0.0.1:37216/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 135591 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 135599 T598 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 135636 T630 oasc.ZkController.register We are http://127.0.0.1:47581/collection1/ and leader is http://127.0.0.1:47581/collection1/
   [junit4]   2> 135636 T630 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:47581
   [junit4]   2> 135636 T630 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 135636 T630 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 135636 T630 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 135638 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 135638 T630 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 135639 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 135639 T624 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47581",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:47581_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 135640 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 135742 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 135742 T645 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 135742 T632 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 136601 T598 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59329_
   [junit4]   2> 136603 T598 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59329_
   [junit4]   2> 136605 T623 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 136605 T632 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 136605 T645 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 136623 T646 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 136624 T646 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 136625 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 136625 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 136625 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 136625 T646 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 136626 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 136626 T624 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59329",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:59329_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 136626 T624 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 136627 T624 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 136683 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 136785 T632 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 136785 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 136785 T645 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 137625 T646 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 137626 T646 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 137626 T646 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 137626 T646 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 137627 T646 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 137627 T646 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-003/collection1/'
   [junit4]   2> 137628 T646 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 137629 T646 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 137629 T646 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 137671 T646 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 137690 T646 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 137701 T646 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 137707 T646 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 137920 T646 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 137922 T646 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 137924 T646 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 137927 T646 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 137958 T646 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 137958 T646 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 137959 T646 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-003/collection1/, dataDir=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty1/
   [junit4]   2> 137959 T646 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1a1b028
   [junit4]   2> 137963 T646 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty1
   [junit4]   2> 137963 T646 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty1/index/
   [junit4]   2> 137963 T646 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 137964 T646 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty1/index
   [junit4]   2> 137964 T646 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.504697298399816]
   [junit4]   2> 137975 T646 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@117a72a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@21ccf6),segFN=segments_1,generation=1}
   [junit4]   2> 137975 T646 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 137979 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 137979 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 137979 T646 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 137980 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 137980 T646 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 137980 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 137980 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 137981 T646 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 137981 T646 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 137982 T646 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 137982 T646 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 137982 T646 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 137982 T646 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 137983 T646 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 137983 T646 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 137984 T646 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 137986 T646 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 137988 T646 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 137988 T646 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 137989 T646 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=13, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 137990 T646 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@117a72a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@21ccf6),segFN=segments_1,generation=1}
   [junit4]   2> 137990 T646 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 137990 T646 oass.SolrIndexSearcher.<init> Opening Searcher@1add27d[collection1] main
   [junit4]   2> 137991 T646 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 137991 T646 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 137992 T646 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 137992 T646 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 137992 T646 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 137992 T646 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 137993 T646 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 137993 T646 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 137994 T646 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 137994 T646 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 137997 T647 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1add27d[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 137999 T646 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 138000 T650 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59329 collection:collection1 shard:shard2
   [junit4]   2> 138000 T598 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 138000 T598 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 138001 T650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 138020 T650 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 138022 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138022 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138022 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138023 T650 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 138023 T650 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1133 name=collection1 org.apache.solr.core.SolrCore@ad3079 url=http://127.0.0.1:59329/collection1 node=127.0.0.1:59329_ C1133_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:59329, core=collection1, node_name=127.0.0.1:59329_}
   [junit4]   2> 138023 T650 C1133 P59329 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59329/collection1/
   [junit4]   2> 138023 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 138023 T650 C1133 P59329 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 138024 T650 C1133 P59329 oasc.SyncStrategy.syncToMe http://127.0.0.1:59329/collection1/ has no replicas
   [junit4]   2> 138024 T650 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59329/collection1/ shard2
   [junit4]   2> 138024 T650 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 138026 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138029 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138031 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138193 T632 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 138193 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 138193 T645 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 138231 T650 oasc.ZkController.register We are http://127.0.0.1:59329/collection1/ and leader is http://127.0.0.1:59329/collection1/
   [junit4]   2> 138231 T650 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59329
   [junit4]   2> 138232 T650 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 138232 T650 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 138232 T650 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 138233 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138233 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138233 T650 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 138233 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138234 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 138235 T624 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59329",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:59329_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 138236 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 138338 T632 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 138338 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 138338 T645 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 138414 T598 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 138418 T598 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 138438 T598 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:56384
   [junit4]   2> 138439 T598 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 138439 T598 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 138440 T598 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-004
   [junit4]   2> 138440 T598 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-004/'
   [junit4]   2> 138456 T598 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-004/solr.xml
   [junit4]   2> 138471 T598 oasc.CoreContainer.<init> New CoreContainer 30759411
   [junit4]   2> 138472 T598 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-004/]
   [junit4]   2> 138472 T598 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 138473 T598 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 138473 T598 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 138473 T598 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 138474 T598 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 138474 T598 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 138475 T598 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 138475 T598 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 138476 T598 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 138477 T598 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 138477 T598 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 138478 T598 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 138478 T598 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37216/solr
   [junit4]   2> 138479 T598 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 138481 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 138483 T661 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76de8e name:ZooKeeperConnection Watcher:127.0.0.1:37216 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 138484 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 138487 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 138489 T663 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37089b name:ZooKeeperConnection Watcher:127.0.0.1:37216/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 138489 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 138495 T598 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 139497 T598 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56384_
   [junit4]   2> 139498 T598 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56384_
   [junit4]   2> 139500 T623 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 139500 T632 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 139500 T663 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 139500 T645 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 139503 T664 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 139504 T664 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 139504 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 139504 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 139505 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 139505 T664 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 139505 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 139506 T624 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56384",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56384_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 139506 T624 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 139506 T624 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 139508 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 139609 T632 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 139609 T645 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 139609 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 139609 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 140505 T664 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 140505 T664 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 140506 T664 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 140506 T664 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 140507 T664 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 140507 T664 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-004/collection1/'
   [junit4]   2> 140508 T664 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 140508 T664 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 140509 T664 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 140548 T664 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 140564 T664 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 140566 T664 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 140573 T664 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 140724 T664 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 140725 T664 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 140726 T664 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 140728 T664 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 140742 T664 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 140742 T664 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 140742 T664 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-004/collection1/, dataDir=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty2/
   [junit4]   2> 140743 T664 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1a1b028
   [junit4]   2> 140743 T664 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty2
   [junit4]   2> 140743 T664 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty2/index/
   [junit4]   2> 140744 T664 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 140744 T664 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty2/index
   [junit4]   2> 140745 T664 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.504697298399816]
   [junit4]   2> 140745 T664 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@27f10e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@333745),segFN=segments_1,generation=1}
   [junit4]   2> 140745 T664 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 140754 T664 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 140755 T664 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 140755 T664 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 140755 T664 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 140756 T664 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 140756 T664 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 140756 T664 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 140757 T664 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 140757 T664 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 140757 T664 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 140758 T664 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 140758 T664 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 140758 T664 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 140759 T664 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 140759 T664 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 140759 T664 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 140762 T664 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 140763 T664 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 140764 T664 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 140764 T664 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=13, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 140766 T664 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@27f10e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@333745),segFN=segments_1,generation=1}
   [junit4]   2> 140766 T664 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 140766 T664 oass.SolrIndexSearcher.<init> Opening Searcher@1385d40[collection1] main
   [junit4]   2> 140766 T664 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 140769 T664 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 140769 T664 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 140770 T664 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 140770 T664 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 140770 T664 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 140771 T664 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 140771 T664 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 140771 T664 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 140772 T664 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 140776 T664 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 140777 T665 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1385d40[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 140778 T668 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:56384 collection:collection1 shard:shard1
   [junit4]   2> 140778 T598 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 140779 T598 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 140802 T668 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 140806 T668 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 140807 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140807 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140807 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140808 T668 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 140808 T668 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1134 name=collection1 org.apache.solr.core.SolrCore@d05cd7 url=http://127.0.0.1:56384/collection1 node=127.0.0.1:56384_ C1134_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:56384, core=collection1, node_name=127.0.0.1:56384_}
   [junit4]   2> 140808 T668 C1134 P56384 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:56384/collection1/
   [junit4]   2> 140808 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 140808 T668 C1134 P56384 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 140809 T668 C1134 P56384 oasc.SyncStrategy.syncToMe http://127.0.0.1:56384/collection1/ has no replicas
   [junit4]   2> 140809 T668 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:56384/collection1/ shard1
   [junit4]   2> 140809 T668 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 140810 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140814 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140817 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140921 T632 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 140921 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 140921 T645 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 140921 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 140964 T668 oasc.ZkController.register We are http://127.0.0.1:56384/collection1/ and leader is http://127.0.0.1:56384/collection1/
   [junit4]   2> 140965 T668 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:56384
   [junit4]   2> 140965 T668 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 140965 T668 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 140965 T668 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 140966 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140966 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140966 T668 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 140967 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 140968 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 140968 T624 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56384",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56384_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 140970 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 141072 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141072 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141072 T645 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141072 T632 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 141145 T598 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 141146 T598 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 141148 T598 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:33472
   [junit4]   2> 141149 T598 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 141149 T598 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 141150 T598 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-005
   [junit4]   2> 141150 T598 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-005/'
   [junit4]   2> 141167 T598 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-005/solr.xml
   [junit4]   2> 141185 T598 oasc.CoreContainer.<init> New CoreContainer 3296752
   [junit4]   2> 141185 T598 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-005/]
   [junit4]   2> 141186 T598 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 141186 T598 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 141187 T598 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 141187 T598 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 141187 T598 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 141188 T598 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 141188 T598 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 141188 T598 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 141189 T598 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 141190 T598 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 141190 T598 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 141190 T598 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 141191 T598 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37216/solr
   [junit4]   2> 141191 T598 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 141192 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 141194 T679 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8db62a name:ZooKeeperConnection Watcher:127.0.0.1:37216 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 141195 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 141197 T598 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 141198 T681 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@921c8e name:ZooKeeperConnection Watcher:127.0.0.1:37216/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 141199 T598 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 141204 T598 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 142206 T598 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33472_
   [junit4]   2> 142208 T598 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33472_
   [junit4]   2> 142210 T623 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142210 T663 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142210 T632 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142210 T645 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142210 T681 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 142214 T682 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 142214 T682 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 142215 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 142215 T682 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 142215 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 142215 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 142216 T624 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 142216 T624 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33472",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33472_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 142217 T624 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 142217 T624 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 142219 T623 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 142320 T623 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 142320 T663 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 142320 T681 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 142320 T632 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 142320 T645 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 143215 T682 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 143216 T682 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 143216 T682 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 143216 T682 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 143217 T682 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 143217 T682 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-005/collection1/'
   [junit4]   2> 143218 T682 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-005/collection1/lib/.svn/' to classloader
   [junit4]   2> 143218 T682 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-005/collection1/lib/classes/' to classloader
   [junit4]   2> 143218 T682 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-005/collection1/lib/README' to classloader
   [junit4]   2> 143247 T682 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 143259 T682 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 143260 T682 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 143264 T682 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 143428 T682 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 143430 T682 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 143430 T682 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143434 T682 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 143483 T682 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection 

[...truncated too long message...]

   [junit4]   2> 311670 T681 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty3
   [junit4]   2> 311670 T681 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty3/index [CachedDir<<refCount=0;path=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty3/index;done=false>>]
   [junit4]   2> 311670 T681 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty3/index
   [junit4]   2> 311670 T681 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 311671 T681 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 311671 T681 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 311671 T681 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1_1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 311672 T681 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> ASYNC  NEW_CORE C1571 name=collection1 org.apache.solr.core.SolrCore@703810 url=http://127.0.0.1:41332/collection1 node=127.0.0.1:41332_ C1571_STATE=coll:collection1 core:collection1 props:{state=active, base_url=http://127.0.0.1:41332, core=collection1, node_name=127.0.0.1:41332_}
   [junit4]   2> 311831 T701 C1571 P41332 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 311831 T701 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 311831 T701 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@703810
   [junit4]   2> 311836 T701 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=94,cumulative_deletesById=19,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=5204,transaction_logs_total_number=2}
   [junit4]   2> 311837 T701 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 311837 T701 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 311837 T701 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 311838 T701 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 311838 T701 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 311838 T701 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty4/index [CachedDir<<refCount=0;path=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty4/index;done=false>>]
   [junit4]   2> 311838 T701 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty4/index
   [junit4]   2> 311838 T701 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty4 [CachedDir<<refCount=0;path=./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty4;done=false>>]
   [junit4]   2> 311838 T701 oasc.CachingDirectoryFactory.close Closing directory: ./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001/tempDir-001/jetty4
   [junit4]   2> 311839 T701 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 311839 T701 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J0/./solr.cloud.ShardSplitTest-8C030E2FB8362E26-001
   [junit4]   2> NOTE: test params are: codec=Lucene46: {_version_=FST41, n_ti=PostingsFormat(name=Memory doPackFST= true), id=PostingsFormat(name=FSTPulsing41)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=zh_HK, timezone=SystemV/AST4
   [junit4]   2> NOTE: Linux 3.8.0-39-generic i386/Oracle Corporation 1.8.0_20-ea (32-bit)/cpus=8,threads=1,free=50945504,total=135786496
   [junit4]   2> NOTE: All tests run in this JVM: [TestPostingsSolrHighlighter, TestSurroundQueryParser, BasicDistributedZkTest, TestSolrIndexConfig, TestSimpleQParserPlugin, DistributedSpellCheckComponentTest, ShardSplitTest]
   [junit4] Completed on J0 in 179.06s, 1 test, 1 error <<< FAILURES!

[...truncated 995 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/common-build.xml:1293: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/common-build.xml:919: There were test failures: 387 suites, 1620 tests, 1 error, 37 ignored (16 assumptions)

Total time: 57 minutes 12 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0_20-ea-b11 -server -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-4.8-Linux (32bit/jdk1.8.0_20-ea-b11) - Build # 118 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.8-Linux/118/
Java: 32bit/jdk1.8.0_20-ea-b11 -client -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.CustomCollectionTest.testDistribSearch

Error Message:
createshard the collection error [Watcher fired on path: null state: SyncConnected type None]

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: createshard the collection error [Watcher fired on path: null state: SyncConnected type None]
	at __randomizedtesting.SeedInfo.seed([143363FA6DA6510B:95D5EDE21AF93137]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:554)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:274)
	at org.apache.solr.cloud.CustomCollectionTest.doTest(CustomCollectionTest.java:127)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11004 lines...]
   [junit4] Suite: org.apache.solr.cloud.CustomCollectionTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/init-core-data-001
   [junit4]   2> 329424 T1340 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 329424 T1340 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /ca_qv/hc
   [junit4]   2> 329428 T1340 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 329428 T1340 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 329429 T1341 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 329529 T1340 oasc.ZkTestServer.run start zk server on port:54710
   [junit4]   2> 329530 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329533 T1347 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28519b name:ZooKeeperConnection Watcher:127.0.0.1:54710 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329533 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329533 T1340 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 329537 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329538 T1349 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17ec045 name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329539 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329539 T1340 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 329542 T1340 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 329544 T1340 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 329546 T1340 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 329548 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 329549 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 329553 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 329553 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 329556 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 329557 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 329559 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 329559 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 329561 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 329562 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 329565 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 329565 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 329568 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 329568 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 329571 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 329571 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 329574 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 329574 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 329576 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 329577 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 329579 T1340 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 329579 T1340 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 329582 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329584 T1351 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c868fa name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329584 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329912 T1340 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 329917 T1340 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 329926 T1340 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:56963
   [junit4]   2> 329929 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 329930 T1340 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 329930 T1340 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-002
   [junit4]   2> 329931 T1340 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-002/'
   [junit4]   2> 329950 T1340 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-002/solr.xml
   [junit4]   2> 329962 T1340 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-002
   [junit4]   2> 329962 T1340 oasc.CoreContainer.<init> New CoreContainer 26581223
   [junit4]   2> 329963 T1340 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-002/]
   [junit4]   2> 329963 T1340 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 329964 T1340 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 329964 T1340 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 329964 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 329965 T1340 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 329965 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 329965 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 329966 T1340 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 329966 T1340 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 329970 T1340 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 329970 T1340 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 329971 T1340 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 329971 T1340 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54710/solr
   [junit4]   2> 329971 T1340 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 329972 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329973 T1362 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@53963f name:ZooKeeperConnection Watcher:127.0.0.1:54710 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329974 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329976 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329977 T1364 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ecefdb name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329978 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329980 T1340 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 329982 T1340 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 329984 T1340 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 329987 T1340 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 329991 T1340 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 329995 T1340 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 329997 T1340 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56963_ca_qv%2Fhc
   [junit4]   2> 329998 T1340 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56963_ca_qv%2Fhc
   [junit4]   2> 330000 T1340 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 330004 T1340 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 330008 T1340 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:56963_ca_qv%2Fhc
   [junit4]   2> 330009 T1340 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 330011 T1340 oasc.Overseer.start Overseer (id=91711654017171460-127.0.0.1:56963_ca_qv%2Fhc-n_0000000000) starting
   [junit4]   2> 330014 T1340 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 330023 T1366 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 330023 T1340 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 330024 T1366 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes at 127.0.0.1:56963_ca_qv%2Fhc
   [junit4]   2> 330026 T1340 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 330028 T1340 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 330034 T1365 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 330035 T1340 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-002
   [junit4]   2> 330047 T1340 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-002/conf/
   [junit4]   2> 330162 T1340 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 330163 T1340 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 330164 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 330168 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 330170 T1369 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e139fc name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 330171 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 330173 T1340 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 330175 T1340 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 330475 T1340 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 330476 T1340 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 330480 T1340 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 330482 T1340 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:55880
   [junit4]   2> 330485 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 330485 T1340 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 330486 T1340 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003
   [junit4]   2> 330486 T1340 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/'
   [junit4]   2> 330504 T1340 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/solr.xml
   [junit4]   2> 330515 T1340 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003
   [junit4]   2> 330516 T1340 oasc.CoreContainer.<init> New CoreContainer 620645
   [junit4]   2> 330516 T1340 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/]
   [junit4]   2> 330517 T1340 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 330517 T1340 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 330517 T1340 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 330518 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 330518 T1340 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 330518 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 330519 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 330519 T1340 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 330519 T1340 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 330522 T1340 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 330522 T1340 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 330523 T1340 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 330523 T1340 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54710/solr
   [junit4]   2> 330523 T1340 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 330524 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 330526 T1380 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1af5367 name:ZooKeeperConnection Watcher:127.0.0.1:54710 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 330526 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 330528 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 330529 T1382 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65280a name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 330529 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 330535 T1340 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 331538 T1340 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55880_ca_qv%2Fhc
   [junit4]   2> 331539 T1340 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55880_ca_qv%2Fhc
   [junit4]   2> 331542 T1369 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 331542 T1364 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 331542 T1382 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 331553 T1340 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003
   [junit4]   2> 331560 T1340 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/conf/
   [junit4]   2> 331691 T1340 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 331693 T1340 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 331693 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 331958 T1340 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 331959 T1340 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 331963 T1340 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 331968 T1340 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:33835
   [junit4]   2> 331971 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 331971 T1340 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 331971 T1340 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004
   [junit4]   2> 331972 T1340 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/'
   [junit4]   2> 331988 T1340 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/solr.xml
   [junit4]   2> 332006 T1340 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004
   [junit4]   2> 332007 T1340 oasc.CoreContainer.<init> New CoreContainer 17718514
   [junit4]   2> 332007 T1340 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/]
   [junit4]   2> 332008 T1340 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 332008 T1340 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 332008 T1340 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 332009 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 332009 T1340 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 332009 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 332010 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 332010 T1340 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 332010 T1340 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 332013 T1340 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 332014 T1340 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 332014 T1340 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 332014 T1340 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54710/solr
   [junit4]   2> 332015 T1340 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 332015 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 332017 T1394 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18b88dc name:ZooKeeperConnection Watcher:127.0.0.1:54710 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 332018 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 332020 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 332022 T1396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2522b0 name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 332022 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 332028 T1340 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 333031 T1340 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33835_ca_qv%2Fhc
   [junit4]   2> 333034 T1340 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33835_ca_qv%2Fhc
   [junit4]   2> 333040 T1364 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 333041 T1396 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 333041 T1382 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 333040 T1369 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 333044 T1340 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004
   [junit4]   2> 333050 T1340 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/conf/
   [junit4]   2> 333161 T1340 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 333163 T1340 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 333164 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 333432 T1340 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 333433 T1340 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 333437 T1340 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 333439 T1340 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:42476
   [junit4]   2> 333441 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 333441 T1340 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 333442 T1340 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005
   [junit4]   2> 333442 T1340 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/'
   [junit4]   2> 333459 T1340 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/solr.xml
   [junit4]   2> 333479 T1340 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005
   [junit4]   2> 333480 T1340 oasc.CoreContainer.<init> New CoreContainer 20429530
   [junit4]   2> 333480 T1340 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/]
   [junit4]   2> 333481 T1340 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 333481 T1340 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 333482 T1340 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 333482 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 333482 T1340 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 333483 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 333483 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 333483 T1340 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 333484 T1340 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 333487 T1340 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 333487 T1340 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 333487 T1340 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 333487 T1340 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54710/solr
   [junit4]   2> 333488 T1340 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 333489 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 333491 T1408 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14a5038 name:ZooKeeperConnection Watcher:127.0.0.1:54710 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 333491 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 333493 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 333494 T1410 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1feeafa name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 333495 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 333501 T1340 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 334504 T1340 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42476_ca_qv%2Fhc
   [junit4]   2> 334505 T1340 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42476_ca_qv%2Fhc
   [junit4]   2> 334508 T1382 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334508 T1369 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334508 T1396 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334508 T1410 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334508 T1364 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334511 T1340 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005
   [junit4]   2> 334517 T1340 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/conf/
   [junit4]   2> 334623 T1340 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 334624 T1340 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 334624 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 334896 T1340 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
   [junit4]   2> 334897 T1340 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 334901 T1340 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 334904 T1340 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:43843
   [junit4]   2> 334906 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 334907 T1340 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 334907 T1340 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006
   [junit4]   2> 334908 T1340 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/'
   [junit4]   2> 334929 T1340 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/solr.xml
   [junit4]   2> 334947 T1340 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006
   [junit4]   2> 334948 T1340 oasc.CoreContainer.<init> New CoreContainer 23288268
   [junit4]   2> 334948 T1340 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/]
   [junit4]   2> 334949 T1340 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 334949 T1340 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 334950 T1340 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 334950 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 334950 T1340 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 334951 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 334951 T1340 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 334951 T1340 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 334952 T1340 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 334954 T1340 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 334954 T1340 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 334955 T1340 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 334955 T1340 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54710/solr
   [junit4]   2> 334955 T1340 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 334956 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 334958 T1422 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f1817 name:ZooKeeperConnection Watcher:127.0.0.1:54710 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 334959 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 334961 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 334962 T1424 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1aad021 name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 334962 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 334967 T1340 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 335969 T1340 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43843_ca_qv%2Fhc
   [junit4]   2> 335970 T1340 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43843_ca_qv%2Fhc
   [junit4]   2> 335975 T1382 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 335975 T1410 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 335975 T1369 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 335975 T1364 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 335975 T1424 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 335975 T1396 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 335979 T1340 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006
   [junit4]   2> 335986 T1340 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/conf/
   [junit4]   2> 336119 T1340 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 336120 T1340 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 336120 T1340 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 336123 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 336124 T1427 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@886c94 name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 336124 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 336125 T1340 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 336128 T1340 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 336130 T1429 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1961a0c name:ZooKeeperConnection Watcher:127.0.0.1:54710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 336131 T1340 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 336132 T1340 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 336222 T1405 oasha.CollectionsHandler.handleCreateAction Creating Collection : replicationFactor=2&shards=a,b,c&maxShardsPerNode=2&router.name=implicit&name=implicitcoll0&action=CREATE&wt=javabin&version=2
   [junit4]   2> 336224 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336225 T1366 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"implicitcoll0",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "shards":"a,b,c",
   [junit4]   2> 	  "router.name":"implicit"}
   [junit4]   2> 336225 T1366 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"implicitcoll0",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "shards":"a,b,c",
   [junit4]   2> 	  "router.name":"implicit"}
   [junit4]   2> 336227 T1366 oasc.OverseerCollectionProcessor.createConfNode Only one config set found in zk - using it:conf1
   [junit4]   2> 336227 T1366 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/implicitcoll0 
   [junit4]   2> 336227 T1366 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 336230 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336231 T1365 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 336232 T1365 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: implicitcoll0
   [junit4]   2> 336232 T1365 oasc.Overseer$ClusterStateUpdater.createCollection Create collection implicitcoll0 with shards [a, b, c]
   [junit4]   2> 336234 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336235 T1410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336235 T1396 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336235 T1382 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336236 T1424 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336236 T1429 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336235 T1427 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336235 T1369 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336235 T1364 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336330 T1366 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection, shardNames [a, b, c] , replicationFactor : 2
   [junit4]   2> 336331 T1366 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_a_replica1 as part of slice a of collection implicitcoll0 on 127.0.0.1:55880_ca_qv%2Fhc
   [junit4]   2> 336332 T1366 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_a_replica2 as part of slice a of collection implicitcoll0 on 127.0.0.1:56963_ca_qv%2Fhc
   [junit4]   2> 336334 T1366 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_b_replica1 as part of slice b of collection implicitcoll0 on 127.0.0.1:42476_ca_qv%2Fhc
   [junit4]   2> 336337 T1366 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_b_replica2 as part of slice b of collection implicitcoll0 on 127.0.0.1:43843_ca_qv%2Fhc
   [junit4]   2> 336345 T1366 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_c_replica1 as part of slice c of collection implicitcoll0 on 127.0.0.1:33835_ca_qv%2Fhc
   [junit4]   2> 336348 T1366 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_c_replica2 as part of slice c of collection implicitcoll0 on 127.0.0.1:55880_ca_qv%2Fhc
   [junit4]   2> 336471 T1420 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=implicitcoll0_b_replica2&action=CREATE&numShards=3&collection=implicitcoll0&shard=b&wt=javabin&version=2
   [junit4]   2> 336472 T1404 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=implicitcoll0_b_replica1&action=CREATE&numShards=3&collection=implicitcoll0&shard=b&wt=javabin&version=2
   [junit4]   2> 336472 T1392 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=implicitcoll0_c_replica1&action=CREATE&numShards=3&collection=implicitcoll0&shard=c&wt=javabin&version=2
   [junit4]   2> 336475 T1392 oasc.ZkController.publish publishing core=implicitcoll0_c_replica1 state=down collection=implicitcoll0
   [junit4]   2> 336475 T1404 oasc.ZkController.publish publishing core=implicitcoll0_b_replica1 state=down collection=implicitcoll0
   [junit4]   2> 336477 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336477 T1420 oasc.ZkController.publish publishing core=implicitcoll0_b_replica2 state=down collection=implicitcoll0
   [junit4]   2> 336478 T1404 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 336478 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336479 T1392 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 336478 T1420 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 336480 T1365 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 336481 T1365 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:33835/ca_qv/hc",
   [junit4]   2> 	  "core":"implicitcoll0_c_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:33835_ca_qv%2Fhc",
   [junit4]   2> 	  "shard":"c",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 336481 T1377 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=implicitcoll0_a_replica1&action=CREATE&numShards=3&collection=implicitcoll0&shard=a&wt=javabin&version=2
   [junit4]   2> 336484 T1377 oasc.ZkController.publish publishing core=implicitcoll0_a_replica1 state=down collection=implicitcoll0
   [junit4]   2> 336484 T1365 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:42476/ca_qv/hc",
   [junit4]   2> 	  "core":"implicitcoll0_b_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42476_ca_qv%2Fhc",
   [junit4]   2> 	  "shard":"b",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 336485 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336485 T1377 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 336487 T1365 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:43843/ca_qv/hc",
   [junit4]   2> 	  "core":"implicitcoll0_b_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43843_ca_qv%2Fhc",
   [junit4]   2> 	  "shard":"b",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 336489 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336490 T1365 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55880/ca_qv/hc",
   [junit4]   2> 	  "core":"implicitcoll0_a_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:55880_ca_qv%2Fhc",
   [junit4]   2> 	  "shard":"a",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 336493 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336498 T1378 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=implicitcoll0_c_replica2&action=CREATE&numShards=3&collection=implicitcoll0&shard=c&wt=javabin&version=2
   [junit4]   2> 336500 T1378 oasc.ZkController.publish publishing core=implicitcoll0_c_replica2 state=down collection=implicitcoll0
   [junit4]   2> 336501 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336501 T1378 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 336503 T1365 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:55880/ca_qv/hc",
   [junit4]   2> 	  "core":"implicitcoll0_c_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:55880_ca_qv%2Fhc",
   [junit4]   2> 	  "shard":"c",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 336504 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336508 T1360 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=implicitcoll0_a_replica2&action=CREATE&numShards=3&collection=implicitcoll0&shard=a&wt=javabin&version=2
   [junit4]   2> 336510 T1360 oasc.ZkController.publish publishing core=implicitcoll0_a_replica2 state=down collection=implicitcoll0
   [junit4]   2> 336510 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336511 T1360 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 336512 T1365 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:56963/ca_qv/hc",
   [junit4]   2> 	  "core":"implicitcoll0_a_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56963_ca_qv%2Fhc",
   [junit4]   2> 	  "shard":"a",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 336513 T1364 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336617 T1410 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336617 T1369 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336617 T1427 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336617 T1424 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336617 T1382 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336617 T1396 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336617 T1429 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336617 T1364 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 337478 T1404 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_b_replica1
   [junit4]   2> 337480 T1420 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_b_replica2
   [junit4]   2> 337480 T1392 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_c_replica1
   [junit4]   2> 337480 T1404 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 337481 T1392 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 337480 T1420 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 337481 T1404 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 337482 T1404 oascc.ZkStateReader.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 337482 T1392 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 337483 T1392 oascc.ZkStateReader.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 337482 T1420 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 337483 T1404 oascc.ZkStateReader.readConfigName path=/collections/implicitcoll0 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 337484 T1392 oascc.ZkStateReader.readConfigName path=/collections/implicitcoll0 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 337483 T1420 oascc.ZkStateReader.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 337484 T1392 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/implicitcoll0_c_replica1/'
   [junit4]   2> 337484 T1404 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/implicitcoll0_b_replica1/'
   [junit4]   2> 337485 T1420 oascc.ZkStateReader.readConfigName path=/collections/implicitcoll0 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 337486 T1420 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/implicitcoll0_b_replica2/'
   [junit4]   2> 337486 T1377 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_a_replica1
   [junit4]   2> 337487 T1377 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 337489 T1377 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 337489 T1377 oascc.ZkStateReader.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 337493 T1377 oascc.ZkStateReader.readConfigName path=/collections/implicitcoll0 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 337494 T1377 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_a_replica1/'
   [junit4]   2> 337511 T1378 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_c_replica2
   [junit4]   2> 337511 T1360 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_a_replica2
   [junit4]   2> 337512 T1378 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 337512 T1360 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 337513 T1378 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 337513 T1378 oascc.ZkStateReader.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 337514 T1360 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 337514 T1360 oascc.ZkStateReader.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 337515 T1378 oascc.ZkStateReader.readConfigName path=/collections/implicitcoll0 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 337516 T1378 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_c_replica2/'
   [junit4]   2> 337516 T1360 oascc.ZkStateReader.readConfigName path=/collections/implicitcoll0 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 337517 T1360 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-002/implicitcoll0_a_replica2/'
   [junit4]   2> 337547 T1392 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 337572 T1420 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 337573 T1404 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 337576 T1392 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 337591 T1392 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 337597 T1404 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 337598 T1420 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 337599 T1378 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 337600 T1420 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 337601 T1404 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 337603 T1360 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 337623 T1420 oass.IndexSchema.readSchema [implicitcoll0_b_replica2] Schema name=test
   [junit4]   2> 337626 T1392 oass.IndexSchema.readSchema [implicitcoll0_c_replica1] Schema name=test
   [junit4]   2> 337626 T1378 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 337623 T1404 oass.IndexSchema.readSchema [implicitcoll0_b_replica1] Schema name=test
   [junit4]   2> 337628 T1377 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_48
   [junit4]   2> 337634 T1360 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 337636 T1378 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 337638 T1360 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 337666 T1377 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 337676 T1377 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 337679 T1378 oass.IndexSchema.readSchema [implicitcoll0_c_replica2] Schema name=test
   [junit4]   2> 337685 T1360 oass.IndexSchema.readSchema [implicitcoll0_a_replica2] Schema name=test
   [junit4]   2> 337691 T1377 oass.IndexSchema.readSchema [implicitcoll0_a_replica1] Schema name=test
   [junit4]   2> 338053 T1404 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338063 T1404 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338064 T1404 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338100 T1392 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338100 T1392 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338101 T1392 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338106 T1420 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338107 T1420 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338108 T1392 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338109 T1404 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338111 T1392 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338112 T1378 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338113 T1378 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338114 T1378 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338116 T1420 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338112 T1404 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338137 T1360 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338138 T1360 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338139 T1420 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338139 T1360 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338141 T1377 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338142 T1377 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 338143 T1377 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338144 T1420 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338145 T1378 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338148 T1378 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338149 T1377 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338150 T1360 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338151 T1377 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338152 T1360 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338159 T1392 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338161 T1404 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338164 T1392 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338167 T1404 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338168 T1392 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338172 T1392 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338175 T1392 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338172 T1404 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338176 T1392 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338178 T1392 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338179 T1404 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338179 T1404 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338179 T1392 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338178 T1378 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338180 T1392 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_c_replica1' using configuration from collection implicitcoll0
   [junit4]   2> 338180 T1404 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338184 T1420 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338184 T1378 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338182 T1377 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338181 T1392 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 338187 T1392 oasc.SolrCore.<init> [implicitcoll0_c_replica1] Opening new SolrCore at ./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/implicitcoll0_c_replica1/, dataDir=./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/implicitcoll0_c_replica1/data/
   [junit4]   2> 338187 T1392 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f12abf
   [junit4]   2> 338189 T1420 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338187 T1404 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338189 T1378 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338191 T1377 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338189 T1392 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/implicitcoll0_c_replica1/data
   [junit4]   2> 338192 T1392 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/implicitcoll0_c_replica1/data/index/
   [junit4]   2> 338193 T1392 oasc.SolrCore.initIndex WARN [implicitcoll0_c_replica1] Solr index directory './solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/implicitcoll0_c_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 338187 T1360 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338194 T1392 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-004/implicitcoll0_c_replica1/data/index
   [junit4]   2> 338193 T1420 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338191 T1378 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338199 T1378 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338200 T1378 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338191 T1404 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338201 T1404 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_b_replica1' using configuration from collection implicitcoll0
   [junit4]   2> 338203 T1404 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 338203 T1404 oasc.SolrCore.<init> [implicitcoll0_b_replica1] Opening new SolrCore at ./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/implicitcoll0_b_replica1/, dataDir=./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/implicitcoll0_b_replica1/data/
   [junit4]   2> 338204 T1404 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f12abf
   [junit4]   2> 338199 T1420 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338206 T1420 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338198 T1392 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=28.173828125, floorSegmentMB=1.3369140625, forceMergeDeletesPctAllowed=8.640846012209686, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39506353244673165
   [junit4]   2> 338197 T1377 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338207 T1404 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/implicitcoll0_b_replica1/data
   [junit4]   2> 338209 T1404 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/implicitcoll0_b_replica1/data/index/
   [junit4]   2> 338209 T1404 oasc.SolrCore.initIndex WARN [implicitcoll0_b_replica1] Solr index directory './solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/implicitcoll0_b_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 338210 T1392 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@10b2b40 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2235f4),segFN=segments_1,generation=1}
   [junit4]   2> 338210 T1392 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 338207 T1420 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338203 T1360 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338201 T1378 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338213 T1378 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338214 T1378 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_c_replica2' using configuration from collection implicitcoll0
   [junit4]   2> 338214 T1378 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 338212 T1420 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338215 T1420 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338215 T1420 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_b_replica2' using configuration from collection implicitcoll0
   [junit4]   2> 338216 T1420 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 338217 T1392 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 338217 T1360 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338210 T1404 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-005/implicitcoll0_b_replica1/data/index
   [junit4]   2> 338209 T1377 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338219 T1377 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338218 T1404 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=28.173828125, floorSegmentMB=1.3369140625, forceMergeDeletesPctAllowed=8.640846012209686, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39506353244673165
   [junit4]   2> 338217 T1392 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 338217 T1420 oasc.SolrCore.<init> [implicitcoll0_b_replica2] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/implicitcoll0_b_replica2/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/implicitcoll0_b_replica2/data/
   [junit4]   2> 338215 T1378 oasc.SolrCore.<init> [implicitcoll0_c_replica2] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_c_replica2/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_c_replica2/data/
   [junit4]   2> 338221 T1420 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f12abf
   [junit4]   2> 338220 T1392 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 338220 T1404 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@b1e31e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a5c0fa),segFN=segments_1,generation=1}
   [junit4]   2> 338222 T1404 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 338219 T1377 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338222 T1420 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/implicitcoll0_b_replica2/data
   [junit4]   2> 338222 T1392 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 338221 T1360 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338221 T1378 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f12abf
   [junit4]   2> 338226 T1360 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338226 T1392 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 338225 T1420 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/implicitcoll0_b_replica2/data/index/
   [junit4]   2> 338224 T1377 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338228 T1377 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338227 T1392 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 338227 T1378 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_c_replica2/data
   [junit4]   2> 338230 T1392 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 338227 T1360 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338230 T1392 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 338231 T1392 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 338230 T1378 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_c_replica2/data/index/
   [junit4]   2> 338229 T1377 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_a_replica1' using configuration from collection implicitcoll0
   [junit4]   2> 338232 T1377 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 338229 T1420 oasc.SolrCore.initIndex WARN [implicitcoll0_b_replica2] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/implicitcoll0_b_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 338229 T1404 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 338234 T1420 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-006/implicitcoll0_b_replica2/data/index
   [junit4]   2> 338233 T1377 oasc.SolrCore.<init> [implicitcoll0_a_replica1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_a_replica1/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_a_replica1/data/
   [junit4]   2> 338232 T1392 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 338232 T1378 oasc.SolrCore.initIndex WARN [implicitcoll0_c_replica2] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_c_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 338232 T1360 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338236 T1360 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338236 T1392 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 338237 T1392 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 338236 T1378 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_c_replica2/data/index
   [junit4]   2> 338235 T1377 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f12abf
   [junit4]   2> 338234 T1420 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=28.173828125, floorSegmentMB=1.3369140625, forceMergeDeletesPctAllowed=8.640846012209686, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39506353244673165
   [junit4]   2> 338239 T1377 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_a_replica1/data
   [junit4]   2> 338234 T1404 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 338239 T1377 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001/tempDir-003/implicitcoll0_a_replica1/data/index/
   [junit4]   2> 338238 T1378 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=21, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=28.173828125, floorSegmentMB=1.3369140625, forceMergeDeletesPctAllowed=8.640846012209686, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.39506353244673165
   [junit4]   2> 338237 T1392 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]  

[...truncated too long message...]

		at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 	Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections
   [junit4]   2> 		at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:40)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:123)
   [junit4]   2> 		... 7 more
   [junit4]   2> 	Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 		at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:209)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:206)
   [junit4]   2> 		at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:206)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:417)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:386)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:373)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:128)
   [junit4]   2> 		at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:34)
   [junit4]   2> 		... 8 more
   [junit4]   2> 	
   [junit4]   2> 452142 T1424 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 452146 T1424 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 452147 T1424 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 452147 T1424 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/implicitcoll0/leader_elect/b/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CustomCollectionTest -Dtests.method=testDistribSearch -Dtests.seed=143363FA6DA6510B -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es_CL -Dtests.timezone=Africa/Blantyre -Dtests.file.encoding=UTF-8
   [junit4] ERROR    123s J1 | CustomCollectionTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: createshard the collection error [Watcher fired on path: null state: SyncConnected type None]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([143363FA6DA6510B:95D5EDE21AF93137]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:554)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
   [junit4]    > 	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:274)
   [junit4]    > 	at org.apache.solr.cloud.CustomCollectionTest.doTest(CustomCollectionTest.java:127)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 452147 T1424 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 452148 T1340 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build/solr-core/test/J1/./solr.cloud.CustomCollectionTest-143363FA6DA6510B-001
   [junit4]   2> 122730 T1339 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene41, sim=DefaultSimilarity, locale=es_CL, timezone=Africa/Blantyre
   [junit4]   2> NOTE: Linux 3.8.0-39-generic i386/Oracle Corporation 1.8.0_20-ea (32-bit)/cpus=8,threads=1,free=104378616,total=197234688
   [junit4]   2> NOTE: All tests run in this JVM: [TestQuerySenderNoQuery, HdfsCollectionsAPIDistributedZkTest, TestCollapseQParserPlugin, TestSchemaResource, FileUtilsTest, HdfsBasicDistributedZk2Test, TestExpandComponent, SolrPluginUtilsTest, TestOmitPositions, ConvertedLegacyTest, PluginInfoTest, TestAtomicUpdateErrorCases, TestRandomFaceting, TestHashPartitioner, TestFuzzyAnalyzedSuggestions, TermVectorComponentDistributedTest, TestAddFieldRealTimeGet, DistribCursorPagingTest, HardAutoCommitTest, TestDistributedSearch, DistributedExpandComponentTest, DateFieldTest, RequestHandlersTest, TestHighFrequencyDictionaryFactory, DeleteReplicaTest, TestRTGBase, ConnectionManagerTest, TestCodecSupport, RAMDirectoryFactoryTest, TestDistribDocBasedVersion, OutputWriterTest, TestAnalyzeInfixSuggestions, XsltUpdateRequestHandlerTest, PreAnalyzedUpdateProcessorTest, TestZkChroot, TestCollationField, TestSerializedLuceneMatchVersion, SSLMigrationTest, TestNoOpRegenerator, TestUpdate, CurrencyFieldXmlFileTest, CustomCollectionTest]
   [junit4] Completed on J1 in 123.01s, 1 test, 1 error <<< FAILURES!

[...truncated 800 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/common-build.xml:1293: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/common-build.xml:919: There were test failures: 387 suites, 1624 tests, 1 error, 37 ignored (16 assumptions)

Total time: 63 minutes 34 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0_20-ea-b11 -client -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-4.8-Linux (64bit/jdk1.7.0_60-ea-b15) - Build # 117 - Still Failing!

Posted by Robert Muir <rm...@apache.org>.
I'll fix this. I'm sure its because of
https://issues.apache.org/jira/browse/LUCENE-5649

So this test should avoid mockrandom mp since it avoids bulk merges
sometimes for better testing.

On Tue, May 6, 2014 at 1:34 PM, Policeman Jenkins Server
<je...@thetaphi.de> wrote:
> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.8-Linux/117/
> Java: 64bit/jdk1.7.0_60-ea-b15 -XX:-UseCompressedOops -XX:+UseSerialGC
>
> 1 tests failed.
> REGRESSION:  org.apache.lucene.index.TestNRTThreads.testNRTThreads
>
> Error Message:
> Captured an uncaught exception in thread: Thread[id=213, name=Lucene Merge Thread #0, state=RUNNABLE, group=TGRP-TestNRTThreads]
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=213, name=Lucene Merge Thread #0, state=RUNNABLE, group=TGRP-TestNRTThreads]
>         at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22:B34D3F55934B6949]:0)
> Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.AssertionError
>         at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22]:0)
>         at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:545)
>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518)
> Caused by: java.lang.AssertionError
>         at org.apache.lucene.util.FailOnNonBulkMergesInfoStream.message(FailOnNonBulkMergesInfoStream.java:37)
>         at org.apache.lucene.index.SegmentMerger.setMatchingSegmentReaders(SegmentMerger.java:297)
>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4132)
>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3728)
>         at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:405)
>         at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:482)
>
>
>
>
> Build Log:
> [...truncated 473 lines...]
>    [junit4] Suite: org.apache.lucene.index.TestNRTThreads
>    [junit4]   2> ماي 07, 2014 1:21:26 ص com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
>    [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #0,6,TGRP-TestNRTThreads]
>    [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.AssertionError
>    [junit4]   2>        at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22]:0)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:545)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518)
>    [junit4]   2> Caused by: java.lang.AssertionError
>    [junit4]   2>        at org.apache.lucene.util.FailOnNonBulkMergesInfoStream.message(FailOnNonBulkMergesInfoStream.java:37)
>    [junit4]   2>        at org.apache.lucene.index.SegmentMerger.setMatchingSegmentReaders(SegmentMerger.java:297)
>    [junit4]   2>        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4132)
>    [junit4]   2>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3728)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:405)
>    [junit4]   2>        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:482)
>    [junit4]   2>
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestNRTThreads -Dtests.method=testNRTThreads -Dtests.seed=28942B4ED2B07F22 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar -Dtests.timezone=Asia/Chungking -Dtests.file.encoding=UTF-8
>    [junit4] ERROR   10.9s J0 | TestNRTThreads.testNRTThreads <<<
>    [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=213, name=Lucene Merge Thread #0, state=RUNNABLE, group=TGRP-TestNRTThreads]
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22:B34D3F55934B6949]:0)
>    [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.AssertionError
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22]:0)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:545)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518)
>    [junit4]    > Caused by: java.lang.AssertionError
>    [junit4]    >        at org.apache.lucene.util.FailOnNonBulkMergesInfoStream.message(FailOnNonBulkMergesInfoStream.java:37)
>    [junit4]    >        at org.apache.lucene.index.SegmentMerger.setMatchingSegmentReaders(SegmentMerger.java:297)
>    [junit4]    >        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4132)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3728)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:405)
>    [junit4]    >        at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:482)
>    [junit4]   2> NOTE: test params are: codec=Lucene46: {extra29=PostingsFormat(name=MockSep), extra39=MockFixedIntBlock(blockSize=1231), extra28=MockFixedIntBlock(blockSize=1231), extra1=PostingsFormat(name=FSTOrdPulsing41), date=MockFixedIntBlock(blockSize=1231), extra5=PostingsFormat(name=FSTOrdPulsing41), extra11=PostingsFormat(name=FSTPulsing41), extra4=PostingsFormat(name=FSTPulsing41), extra32=PostingsFormat(name=MockSep), extra38=PostingsFormat(name=FSTOrdPulsing41), extra23=PostingsFormat(name=FSTOrdPulsing41), extra0=PostingsFormat(name=FSTPulsing41), extra26=PostingsFormat(name=FSTPulsing41), title=PostingsFormat(name=FSTPulsing41), extra27=PostingsFormat(name=FSTOrdPulsing41), extra37=PostingsFormat(name=FSTPulsing41), extra8=PostingsFormat(name=FSTPulsing41), extra19=PostingsFormat(name=FSTPulsing41), titleTokenized=PostingsFormat(name=MockSep), extra12=PostingsFormat(name=FSTOrdPulsing41), extra33=PostingsFormat(name=FSTPulsing41), extra34=PostingsFormat(name=FSTOrdPulsing41), extra14=PostingsFormat(name=MockSep), extra3=PostingsFormat(name=MockSep), extra13=MockFixedIntBlock(blockSize=1231), extra17=MockFixedIntBlock(blockSize=1231), extra7=PostingsFormat(name=MockSep), packID=PostingsFormat(name=FSTPulsing41), extra22=PostingsFormat(name=FSTPulsing41), extra10=PostingsFormat(name=MockSep), extra9=PostingsFormat(name=FSTOrdPulsing41), extra36=PostingsFormat(name=MockSep), extra20=MockFixedIntBlock(blockSize=1231), extra2=MockFixedIntBlock(blockSize=1231), extra18=PostingsFormat(name=MockSep), extra21=PostingsFormat(name=MockSep), extra31=MockFixedIntBlock(blockSize=1231), extra35=MockFixedIntBlock(blockSize=1231), extra16=PostingsFormat(name=FSTOrdPulsing41), extra30=PostingsFormat(name=FSTOrdPulsing41), extra25=PostingsFormat(name=MockSep), extra6=MockFixedIntBlock(blockSize=1231), docid=PostingsFormat(name=FSTOrdPulsing41), body=MockFixedIntBlock(blockSize=1231), extra15=PostingsFormat(name=FSTPulsing41), extra24=MockFixedIntBlock(blockSize=1231)}, docValues:{titleDV=DocValuesFormat(name=Lucene45)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {extra29=DFR I(ne)BZ(0.3), extra39=IB SPL-D2, extra28=IB SPL-DZ(0.3), extra1=DFR I(n)LZ(0.3), extra11=DFR I(ne)3(800.0), extra5=DFR I(ne)2, extra4=IB LL-D1, extra32=DFR I(n)1, extra38=LM Jelinek-Mercer(0.700000), extra26=IB LL-LZ(0.3), extra0=DFR GB1, extra23=IB SPL-L3(800.0), extra27=DFR I(F)BZ(0.3), extra37=IB SPL-L1, extra8=DFR GB2, extra19=IB SPL-L1, titleTokenized=DFR I(F)2, extra12=DFR I(n)B3(800.0), extra33=IB LL-D3(800.0), extra14=IB SPL-D3(800.0), extra34=DFR I(ne)L3(800.0), extra3=IB LL-L2, extra17=DFR I(ne)B2, extra13=IB LL-L1, extra7=DFR I(n)2, packID=DFR I(n)L1, extra22=DFR GBZ(0.3), extra10=IB LL-DZ(0.3), extra9=IB SPL-L2, extra36=DFR I(n)L2, extra20=DFR I(ne)L2, extra2=DFR GB3(800.0), extra18=DFR I(n)L2, extra21=DFR GLZ(0.3), extra31=DFR I(ne)LZ(0.3), extra35=DFR I(ne)B2, extra16=DFR I(ne)L3(800.0), extra30=DFR GL1, docid=DFR I(ne)BZ(0.3), extra6=DFR I(n)B1, extra25=DFR G1, body=DFR GBZ(0.3), extra15=IB LL-D3(800.0), extra24=DFR I(F)3(800.0)}, locale=ar, timezone=Asia/Chungking
>    [junit4]   2> NOTE: Linux 3.8.0-39-generic amd64/Oracle Corporation 1.7.0_60-ea (64-bit)/cpus=8,threads=1,free=24146832,total=126418944
>    [junit4]   2> NOTE: All tests run in this JVM: [TestCrash, TestStressIndexing2, TestPriorityQueue, TestQueryBuilder, TestPostingsOffsets, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, TestSloppyPhraseQuery, TestBagOfPositions, TestLucene3xStoredFieldsFormat, TestMergeSchedulerExternal, TestSimilarityProvider, TestRamUsageEstimator, TestDocValuesWithThreads, Test2BTerms, TestStressNRT, TestFilteredQuery, TestSpanFirstQuery, Test2BPostingsBytes, TestRegexpRandom, TestNRTThreads]
>    [junit4] Completed on J0 in 10.88s, 1 test, 1 error <<< FAILURES!
>
> [...truncated 1117 lines...]
> BUILD FAILED
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:467: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:447: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:45: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/extra-targets.xml:37: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/build.xml:49: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/common-build.xml:1293: The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/common-build.xml:919: There were test failures: 397 suites, 2906 tests, 1 error, 87 ignored (74 assumptions)
>
> Total time: 14 minutes 50 seconds
> Build step 'Invoke Ant' marked build as failure
> Description set: Java: 64bit/jdk1.7.0_60-ea-b15 -XX:-UseCompressedOops -XX:+UseSerialGC
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-4.8-Linux (64bit/jdk1.7.0_60-ea-b15) - Build # 117 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.8-Linux/117/
Java: 64bit/jdk1.7.0_60-ea-b15 -XX:-UseCompressedOops -XX:+UseSerialGC

1 tests failed.
REGRESSION:  org.apache.lucene.index.TestNRTThreads.testNRTThreads

Error Message:
Captured an uncaught exception in thread: Thread[id=213, name=Lucene Merge Thread #0, state=RUNNABLE, group=TGRP-TestNRTThreads]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=213, name=Lucene Merge Thread #0, state=RUNNABLE, group=TGRP-TestNRTThreads]
	at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22:B34D3F55934B6949]:0)
Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22]:0)
	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:545)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518)
Caused by: java.lang.AssertionError
	at org.apache.lucene.util.FailOnNonBulkMergesInfoStream.message(FailOnNonBulkMergesInfoStream.java:37)
	at org.apache.lucene.index.SegmentMerger.setMatchingSegmentReaders(SegmentMerger.java:297)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4132)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3728)
	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:405)
	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:482)




Build Log:
[...truncated 473 lines...]
   [junit4] Suite: org.apache.lucene.index.TestNRTThreads
   [junit4]   2> ماي 07, 2014 1:21:26 ص com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Lucene Merge Thread #0,6,TGRP-TestNRTThreads]
   [junit4]   2> org.apache.lucene.index.MergePolicy$MergeException: java.lang.AssertionError
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22]:0)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:545)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518)
   [junit4]   2> Caused by: java.lang.AssertionError
   [junit4]   2> 	at org.apache.lucene.util.FailOnNonBulkMergesInfoStream.message(FailOnNonBulkMergesInfoStream.java:37)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.setMatchingSegmentReaders(SegmentMerger.java:297)
   [junit4]   2> 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4132)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3728)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:405)
   [junit4]   2> 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:482)
   [junit4]   2> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestNRTThreads -Dtests.method=testNRTThreads -Dtests.seed=28942B4ED2B07F22 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar -Dtests.timezone=Asia/Chungking -Dtests.file.encoding=UTF-8
   [junit4] ERROR   10.9s J0 | TestNRTThreads.testNRTThreads <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=213, name=Lucene Merge Thread #0, state=RUNNABLE, group=TGRP-TestNRTThreads]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22:B34D3F55934B6949]:0)
   [junit4]    > Caused by: org.apache.lucene.index.MergePolicy$MergeException: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([28942B4ED2B07F22]:0)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.handleMergeException(ConcurrentMergeScheduler.java:545)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:518)
   [junit4]    > Caused by: java.lang.AssertionError
   [junit4]    > 	at org.apache.lucene.util.FailOnNonBulkMergesInfoStream.message(FailOnNonBulkMergesInfoStream.java:37)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.setMatchingSegmentReaders(SegmentMerger.java:297)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4132)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3728)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:405)
   [junit4]    > 	at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:482)
   [junit4]   2> NOTE: test params are: codec=Lucene46: {extra29=PostingsFormat(name=MockSep), extra39=MockFixedIntBlock(blockSize=1231), extra28=MockFixedIntBlock(blockSize=1231), extra1=PostingsFormat(name=FSTOrdPulsing41), date=MockFixedIntBlock(blockSize=1231), extra5=PostingsFormat(name=FSTOrdPulsing41), extra11=PostingsFormat(name=FSTPulsing41), extra4=PostingsFormat(name=FSTPulsing41), extra32=PostingsFormat(name=MockSep), extra38=PostingsFormat(name=FSTOrdPulsing41), extra23=PostingsFormat(name=FSTOrdPulsing41), extra0=PostingsFormat(name=FSTPulsing41), extra26=PostingsFormat(name=FSTPulsing41), title=PostingsFormat(name=FSTPulsing41), extra27=PostingsFormat(name=FSTOrdPulsing41), extra37=PostingsFormat(name=FSTPulsing41), extra8=PostingsFormat(name=FSTPulsing41), extra19=PostingsFormat(name=FSTPulsing41), titleTokenized=PostingsFormat(name=MockSep), extra12=PostingsFormat(name=FSTOrdPulsing41), extra33=PostingsFormat(name=FSTPulsing41), extra34=PostingsFormat(name=FSTOrdPulsing41), extra14=PostingsFormat(name=MockSep), extra3=PostingsFormat(name=MockSep), extra13=MockFixedIntBlock(blockSize=1231), extra17=MockFixedIntBlock(blockSize=1231), extra7=PostingsFormat(name=MockSep), packID=PostingsFormat(name=FSTPulsing41), extra22=PostingsFormat(name=FSTPulsing41), extra10=PostingsFormat(name=MockSep), extra9=PostingsFormat(name=FSTOrdPulsing41), extra36=PostingsFormat(name=MockSep), extra20=MockFixedIntBlock(blockSize=1231), extra2=MockFixedIntBlock(blockSize=1231), extra18=PostingsFormat(name=MockSep), extra21=PostingsFormat(name=MockSep), extra31=MockFixedIntBlock(blockSize=1231), extra35=MockFixedIntBlock(blockSize=1231), extra16=PostingsFormat(name=FSTOrdPulsing41), extra30=PostingsFormat(name=FSTOrdPulsing41), extra25=PostingsFormat(name=MockSep), extra6=MockFixedIntBlock(blockSize=1231), docid=PostingsFormat(name=FSTOrdPulsing41), body=MockFixedIntBlock(blockSize=1231), extra15=PostingsFormat(name=FSTPulsing41), extra24=MockFixedIntBlock(blockSize=1231)}, docValues:{titleDV=DocValuesFormat(name=Lucene45)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {extra29=DFR I(ne)BZ(0.3), extra39=IB SPL-D2, extra28=IB SPL-DZ(0.3), extra1=DFR I(n)LZ(0.3), extra11=DFR I(ne)3(800.0), extra5=DFR I(ne)2, extra4=IB LL-D1, extra32=DFR I(n)1, extra38=LM Jelinek-Mercer(0.700000), extra26=IB LL-LZ(0.3), extra0=DFR GB1, extra23=IB SPL-L3(800.0), extra27=DFR I(F)BZ(0.3), extra37=IB SPL-L1, extra8=DFR GB2, extra19=IB SPL-L1, titleTokenized=DFR I(F)2, extra12=DFR I(n)B3(800.0), extra33=IB LL-D3(800.0), extra14=IB SPL-D3(800.0), extra34=DFR I(ne)L3(800.0), extra3=IB LL-L2, extra17=DFR I(ne)B2, extra13=IB LL-L1, extra7=DFR I(n)2, packID=DFR I(n)L1, extra22=DFR GBZ(0.3), extra10=IB LL-DZ(0.3), extra9=IB SPL-L2, extra36=DFR I(n)L2, extra20=DFR I(ne)L2, extra2=DFR GB3(800.0), extra18=DFR I(n)L2, extra21=DFR GLZ(0.3), extra31=DFR I(ne)LZ(0.3), extra35=DFR I(ne)B2, extra16=DFR I(ne)L3(800.0), extra30=DFR GL1, docid=DFR I(ne)BZ(0.3), extra6=DFR I(n)B1, extra25=DFR G1, body=DFR GBZ(0.3), extra15=IB LL-D3(800.0), extra24=DFR I(F)3(800.0)}, locale=ar, timezone=Asia/Chungking
   [junit4]   2> NOTE: Linux 3.8.0-39-generic amd64/Oracle Corporation 1.7.0_60-ea (64-bit)/cpus=8,threads=1,free=24146832,total=126418944
   [junit4]   2> NOTE: All tests run in this JVM: [TestCrash, TestStressIndexing2, TestPriorityQueue, TestQueryBuilder, TestPostingsOffsets, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, TestSloppyPhraseQuery, TestBagOfPositions, TestLucene3xStoredFieldsFormat, TestMergeSchedulerExternal, TestSimilarityProvider, TestRamUsageEstimator, TestDocValuesWithThreads, Test2BTerms, TestStressNRT, TestFilteredQuery, TestSpanFirstQuery, Test2BPostingsBytes, TestRegexpRandom, TestNRTThreads]
   [junit4] Completed on J0 in 10.88s, 1 test, 1 error <<< FAILURES!

[...truncated 1117 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/build.xml:49: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/common-build.xml:1293: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.8-Linux/lucene/common-build.xml:919: There were test failures: 397 suites, 2906 tests, 1 error, 87 ignored (74 assumptions)

Total time: 14 minutes 50 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0_60-ea-b15 -XX:-UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure