You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/02/17 17:38:30 UTC

[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.6.0_45) - Build # 9400 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/9400/
Java: 64bit/jdk1.6.0_45 -XX:+UseCompressedOops -XX:+UseSerialGC

2 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
Captured an uncaught exception in thread: Thread[id=2416, name=RecoveryThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=2416, name=RecoveryThread, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
	at __randomizedtesting.SeedInfo.seed([A1E80C6C0C61B9C5:200E82747B3ED9F9]:0)
Caused by: java.lang.AssertionError: file _0_1.del does not exist
	at __randomizedtesting.SeedInfo.seed([A1E80C6C0C61B9C5]:0)
	at org.apache.lucene.index.IndexWriter.filesExist(IndexWriter.java:4363)
	at org.apache.lucene.index.IndexWriter.startCommit(IndexWriter.java:4429)
	at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:2926)
	at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3022)
	at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:2989)
	at org.apache.solr.update.DirectUpdateHandler2.commit(DirectUpdateHandler2.java:578)
	at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:396)
	at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:247)


REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
deletealias the collection time out:180s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: deletealias the collection time out:180s
	at __randomizedtesting.SeedInfo.seed([A1E80C6C0C61B9C5:200E82747B3ED9F9]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:495)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.CollectionAdminRequest.process(CollectionAdminRequest.java:271)
	at org.apache.solr.client.solrj.request.CollectionAdminRequest.deleteAlias(CollectionAdminRequest.java:402)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testSolrJAPICalls(CollectionsAPIDistributedZkTest.java:304)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:199)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:597)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:662)




Build Log:
[...truncated 9604 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> 418628 T1180 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 418629 T1180 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 418632 T1180 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solrtest-CollectionsAPIDistributedZkTest-1392653916300
   [junit4]   2> 418633 T1180 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 418633 T1181 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 418733 T1180 oasc.ZkTestServer.run start zk server on port:40738
   [junit4]   2> 418736 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 418816 T1187 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4181c037 name:ZooKeeperConnection Watcher:127.0.0.1:40738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 418817 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 418817 T1180 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 418827 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 418829 T1189 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f0610f5 name:ZooKeeperConnection Watcher:127.0.0.1:40738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 418830 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 418830 T1180 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 418839 T1180 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 418846 T1180 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 418849 T1180 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 418853 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 418854 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 418859 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 418860 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 418864 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 418865 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 418869 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 418870 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 418874 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 418875 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 418878 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 418879 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 418882 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 418883 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 418888 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 418888 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 418892 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 418893 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 418897 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 418898 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 418901 T1180 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 418902 T1180 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> Using new style solr.xml
   [junit4]   2> 418983 T1180 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 418986 T1180 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:59348
   [junit4]   2> 418986 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 418987 T1180 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 418987 T1180 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575
   [junit4]   2> 418987 T1180 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/'
   [junit4]   2> 418997 T1180 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/solr.xml
   [junit4]   2> 419032 T1180 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575
   [junit4]   2> 419032 T1180 oasc.CoreContainer.<init> New CoreContainer 1010383210
   [junit4]   2> 419032 T1180 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/]
   [junit4]   2> 419034 T1180 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 419034 T1180 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 419034 T1180 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 419034 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 419035 T1180 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 419035 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 419035 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 419036 T1180 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 419036 T1180 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 419038 T1180 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 419038 T1180 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 419039 T1180 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 419039 T1180 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:40738/solr
   [junit4]   2> 419040 T1180 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 419040 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 419042 T1200 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50e87d09 name:ZooKeeperConnection Watcher:127.0.0.1:40738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 419042 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 419045 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 419046 T1202 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66e207d1 name:ZooKeeperConnection Watcher:127.0.0.1:40738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 419046 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 419048 T1180 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 419052 T1180 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 419057 T1180 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 419059 T1180 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59348_
   [junit4]   2> 419060 T1180 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59348_
   [junit4]   2> 419064 T1180 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 419067 T1180 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 419074 T1180 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:59348_
   [junit4]   2> 419074 T1180 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 419086 T1180 oasc.Overseer.start Overseer (id=91268967058767875-127.0.0.1:59348_-n_0000000000) starting
   [junit4]   2> 419091 T1180 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 419096 T1204 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 419097 T1180 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 419097 T1204 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 419101 T1180 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 419104 T1180 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 419108 T1203 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 419109 T1180 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575
   [junit4]   2> 419110 T1180 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/conf/
   [junit4]   2> 419142 T1180 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 419142 T1180 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 419143 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 419144 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 419145 T1207 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f62dd7d name:ZooKeeperConnection Watcher:127.0.0.1:40738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 419146 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 419147 T1180 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 419151 T1180 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 419222 T1180 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 419223 T1180 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 419225 T1180 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:43365
   [junit4]   2> 419225 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 419226 T1180 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 419226 T1180 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819
   [junit4]   2> 419226 T1180 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/'
   [junit4]   2> 419236 T1180 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/solr.xml
   [junit4]   2> 419269 T1180 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819
   [junit4]   2> 419270 T1180 oasc.CoreContainer.<init> New CoreContainer 401880071
   [junit4]   2> 419270 T1180 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/]
   [junit4]   2> 419271 T1180 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 419271 T1180 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 419272 T1180 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 419272 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 419272 T1180 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 419273 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 419273 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 419273 T1180 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 419274 T1180 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 419276 T1180 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 419276 T1180 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 419276 T1180 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 419277 T1180 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:40738/solr
   [junit4]   2> 419277 T1180 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 419278 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 419279 T1218 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2183c3da name:ZooKeeperConnection Watcher:127.0.0.1:40738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 419280 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 419282 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 419283 T1220 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57ff0352 name:ZooKeeperConnection Watcher:127.0.0.1:40738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 419284 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 419290 T1180 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 420294 T1180 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43365_
   [junit4]   2> 420296 T1180 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43365_
   [junit4]   2> 420300 T1202 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 420301 T1207 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 420301 T1220 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 420308 T1180 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819
   [junit4]   2> 420310 T1180 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/conf/
   [junit4]   2> 420350 T1180 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 420351 T1180 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 420351 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 420424 T1180 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 420424 T1180 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 420427 T1180 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:41460
   [junit4]   2> 420428 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 420428 T1180 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 420428 T1180 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019
   [junit4]   2> 420429 T1180 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/'
   [junit4]   2> 420446 T1180 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/solr.xml
   [junit4]   2> 420495 T1180 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019
   [junit4]   2> 420496 T1180 oasc.CoreContainer.<init> New CoreContainer 620274786
   [junit4]   2> 420496 T1180 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/]
   [junit4]   2> 420497 T1180 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 420498 T1180 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 420498 T1180 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 420498 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 420499 T1180 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 420499 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 420499 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 420499 T1180 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 420500 T1180 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 420502 T1180 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 420502 T1180 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 420502 T1180 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 420503 T1180 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:40738/solr
   [junit4]   2> 420503 T1180 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 420504 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 420505 T1232 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ff030e4 name:ZooKeeperConnection Watcher:127.0.0.1:40738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 420506 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 420508 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 420510 T1234 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e44174e name:ZooKeeperConnection Watcher:127.0.0.1:40738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 420510 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 420517 T1180 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 421524 T1180 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41460_
   [junit4]   2> 421527 T1180 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41460_
   [junit4]   2> 421537 T1207 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 421537 T1234 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 421538 T1220 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 421539 T1202 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 421547 T1180 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019
   [junit4]   2> 421548 T1180 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/conf/
   [junit4]   2> 421587 T1180 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 421587 T1180 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 421588 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 421659 T1180 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 421660 T1180 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 421671 T1180 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:48095
   [junit4]   2> 421672 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 421672 T1180 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 421673 T1180 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255
   [junit4]   2> 421674 T1180 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/'
   [junit4]   2> 421687 T1180 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/solr.xml
   [junit4]   2> 421721 T1180 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255
   [junit4]   2> 421722 T1180 oasc.CoreContainer.<init> New CoreContainer 1289467625
   [junit4]   2> 421722 T1180 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/]
   [junit4]   2> 421723 T1180 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 421723 T1180 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 421723 T1180 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 421724 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 421724 T1180 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 421724 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 421725 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 421725 T1180 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 421725 T1180 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 421727 T1180 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 421727 T1180 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 421728 T1180 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 421728 T1180 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:40738/solr
   [junit4]   2> 421728 T1180 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 421729 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 421731 T1246 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6cc5a26a name:ZooKeeperConnection Watcher:127.0.0.1:40738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 421731 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 421734 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 421735 T1248 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@206507d3 name:ZooKeeperConnection Watcher:127.0.0.1:40738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 421735 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 421744 T1180 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 422751 T1180 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48095_
   [junit4]   2> 422753 T1180 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48095_
   [junit4]   2> 422759 T1234 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 422759 T1207 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 422759 T1248 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 422760 T1220 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 422761 T1202 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 422766 T1180 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255
   [junit4]   2> 422767 T1180 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/conf/
   [junit4]   2> 422791 T1180 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 422792 T1180 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 422792 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 422864 T1180 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
   [junit4]   2> 422865 T1180 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 422867 T1180 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:58695
   [junit4]   2> 422867 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 422868 T1180 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 422868 T1180 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1392653920459
   [junit4]   2> 422869 T1180 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1392653920459/'
   [junit4]   2> 422879 T1180 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1392653920459/solr.xml
   [junit4]   2> 422915 T1180 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1392653920459
   [junit4]   2> 422916 T1180 oasc.CoreContainer.<init> New CoreContainer 337674819
   [junit4]   2> 422916 T1180 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1392653920459/]
   [junit4]   2> 422917 T1180 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 422917 T1180 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 422918 T1180 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 422918 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 422918 T1180 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 422919 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 422919 T1180 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 422919 T1180 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 422920 T1180 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 422922 T1180 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 422922 T1180 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 422923 T1180 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 422923 T1180 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:40738/solr
   [junit4]   2> 422923 T1180 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 422924 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 422926 T1260 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62480707 name:ZooKeeperConnection Watcher:127.0.0.1:40738 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 422926 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 422929 T1180 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 422931 T1262 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@363b4064 name:ZooKeeperConnection Watcher:127.0.0.1:40738/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 422931 T1180 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 422940 T1180 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 423945 T1180 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58695_
   [junit4]   2> 423947 T1180 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58695_
   [junit4]   2> 423956 T1248 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 423956 T1234 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 423957 T1202 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 423958 T1207 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 423958 T1220 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 423962 T1262 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 423967 T1180 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1392653920459
   [junit4]   2> 423968 T1180 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1392653920459/conf/
   [junit4]   2> 423998 T1180 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 423999 T1180 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 423999 T1180 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 424010 T1214 oasha.CollectionsHandler.handleCreateAction Creating Collection : numShards=2&collection.configName=conf1&name=solrj_collection&replicationFactor=2&action=CREATE&router.field=myOwnField&wt=javabin&router.name=compositeId&version=2
   [junit4]   2> 424012 T1202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 424014 T1202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 424015 T1204 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 424015 T1204 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 424015 T1204 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
   [junit4]   2> 424015 T1204 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
   [junit4]   2> 424053 T1202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 424055 T1203 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 424056 T1203 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
   [junit4]   2> 424057 T1203 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
   [junit4]   2> 424062 T1202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 424065 T1234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424065 T1202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424065 T1248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424066 T1220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424065 T1207 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424066 T1262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424153 T1204 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 424154 T1204 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:41460_
   [junit4]   2> 424155 T1204 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:48095_
   [junit4]   2> 424158 T1204 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:43365_
   [junit4]   2> 424159 T1228 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down
   [junit4]   2> 424159 T1244 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down
   [junit4]   2> 424159 T1204 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:59348_
   [junit4]   2> 424161 T1216 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down
   [junit4]   2> 424162 T1228 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 424162 T1244 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 424162 T1202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 424163 T1216 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 424164 T1196 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down
   [junit4]   2> 424166 T1202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 424167 T1203 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 424168 T1203 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41460",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41460_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 424169 T1196 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 424175 T1203 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48095",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48095_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 424180 T1202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 424185 T1203 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43365",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43365_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 424190 T1202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 424193 T1203 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59348",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:59348_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 424197 T1202 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 424301 T1207 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424302 T1248 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424302 T1202 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424302 T1220 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424303 T1234 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 424303 T1262 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 425162 T1228 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
   [junit4]   2> 425162 T1244 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
   [junit4]   2> 425164 T1216 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
   [junit4]   2> 425164 T1228 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/solrj_collection_shard1_replica1
   [junit4]   2> 425164 T1244 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/solrj_collection_shard1_replica2
   [junit4]   2> 425165 T1244 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 425164 T1216 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/solrj_collection_shard2_replica1
   [junit4]   2> 425164 T1228 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 425166 T1216 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 425167 T1216 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 425167 T1216 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 425169 T1244 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 425169 T1244 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 425170 T1196 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
   [junit4]   2> 425170 T1228 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 425171 T1228 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 425171 T1196 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/solrj_collection_shard2_replica2
   [junit4]   2> 425171 T1196 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 425174 T1216 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 425174 T1196 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 425174 T1216 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/solrj_collection_shard2_replica1/'
   [junit4]   2> 425174 T1196 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 425178 T1228 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 425179 T1228 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/solrj_collection_shard1_replica1/'
   [junit4]   2> 425180 T1244 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 425181 T1244 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/solrj_collection_shard1_replica2/'
   [junit4]   2> 425181 T1196 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 425184 T1196 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/solrj_collection_shard2_replica2/'
   [junit4]   2> 425284 T1244 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 425289 T1228 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 425298 T1216 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 425319 T1196 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 425398 T1196 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 425401 T1196 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 425402 T1228 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 425405 T1228 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 425412 T1196 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
   [junit4]   2> 425416 T1228 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
   [junit4]   2> 425419 T1244 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 425422 T1244 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 425446 T1244 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
   [junit4]   2> 425458 T1216 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 425460 T1216 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 425495 T1216 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
   [junit4]   2> 426531 T1228 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 426589 T1228 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 426590 T1228 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 426593 T1244 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 426594 T1244 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 426595 T1244 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 426600 T1228 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 426604 T1228 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 426606 T1244 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 426610 T1196 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 426610 T1244 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 426610 T1196 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 426611 T1196 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 426617 T1196 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 426620 T1196 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 426670 T1244 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 426673 T1216 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 426673 T1216 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 426674 T1216 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 426676 T1244 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 426678 T1228 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 426680 T1196 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 426692 T1244 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 426693 T1228 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 426694 T1216 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 426698 T1216 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 426700 T1228 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 426702 T1228 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 426703 T1244 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 426703 T1244 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 426703 T1196 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 426703 T1228 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 426705 T1228 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 426704 T1244 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 426708 T1244 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 426708 T1244 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 426709 T1244 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 426709 T1244 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/solrj_collection_shard1_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/solrj_collection_shard1_replica2/data/
   [junit4]   2> 426710 T1244 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@bc10d02
   [junit4]   2> 426716 T1244 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/solrj_collection_shard1_replica2/data
   [junit4]   2> 426716 T1244 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/solrj_collection_shard1_replica2/data/index/
   [junit4]   2> 426717 T1244 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 426717 T1244 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1392653919255/solrj_collection_shard1_replica2/data/index
   [junit4]   2> 426718 T1244 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=1.8603515625, floorSegmentMB=1.1904296875, forceMergeDeletesPctAllowed=17.8928489670199, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7490457491565093
   [junit4]   2> 426719 T1244 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4f5204ca lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@40131d31),segFN=segments_1,generation=1}
   [junit4]   2> 426720 T1244 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 426721 T1196 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 426724 T1228 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 426725 T1228 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 426725 T1244 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 426725 T1228 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 426726 T1244 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 426726 T1228 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/solrj_collection_shard1_replica1/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/solrj_collection_shard1_replica1/data/
   [junit4]   2> 426726 T1244 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 426730 T1228 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@bc10d02
   [junit4]   2> 426740 T1196 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 426742 T1196 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 426742 T1196 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 426740 T1244 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 426741 T1228 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/solrj_collection_shard1_replica1/data
   [junit4]   2> 426746 T1196 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 426747 T1196 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 426748 T1196 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 426748 T1196 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/solrj_collection_shard2_replica2/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/solrj_collection_shard2_replica2/data/
   [junit4]   2> 426748 T1196 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@bc10d02
   [junit4]   2> 426744 T1244 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 426749 T1196 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/solrj_collection_shard2_replica2/data
   [junit4]   2> 426747 T1228 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/solrj_collection_shard1_replica1/data/index/
   [junit4]   2> 426750 T1196 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/solrj_collection_shard2_replica2/data/index/
   [junit4]   2> 426749 T1244 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 426750 T1196 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 426750 T1228 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 426751 T1196 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1392653916575/solrj_collection_shard2_replica2/data/index
   [junit4]   2> 426751 T1244 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 426752 T1196 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=1.8603515625, floorSegmentMB=1.1904296875, forceMergeDeletesPctAllowed=17.8928489670199, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7490457491565093
   [junit4]   2> 426752 T1228 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1392653918019/solrj_collection_shard1_replica1/data/index
   [junit4]   2> 426753 T1228 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=1.8603515625, floorSegmentMB=1.1904296875, forceMergeDeletesPctAllowed=17.8928489670199, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7490457491565093
   [junit4]   2> 426752 T1244 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 426753 T1196 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5868a774 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@606f3b45),segFN=segments_1,generation=1}
   [junit4]   2> 426754 T1228 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@37f13bed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@14c6731),segFN=segments_1,generation=1}
   [junit4]   2> 426755 T1228 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 426754 T1244 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 426756 T1244 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 426756 T1244 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 426757 T1244 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 426757 T1244 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 426758 T1244 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 426758 T1244 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 426759 T1244 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 426754 T1196 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 426766 T1196 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 426767 T1228 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 426768 T1228 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 426768 T1228 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 426769 T1228 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 426769 T1228 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 426767 T1196 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 426769 T1228 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 426769 T1244 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 426770 T1228 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 426771 T1228 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 426771 T1228 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 426772 T1228 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 426773 T1228 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 426773 T1228 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 426770 T1196 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 426773 T1228 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 426774 T1228 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 426774 T1196 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 426775 T1228 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 426774 T1244 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 426775 T1228 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 426775 T1196 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 426776 T1196 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 426777 T1196 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 426777 T1196 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 426778 T1196 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 426778 T1196 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 426779 T1196 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 426779 T1196 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 426779 T1196 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 426780 T1196 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 426780 T1196 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 426780 T1196 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 426776 T1244 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 426782 T1216 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 426777 T1228 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 426788 T1196 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 426791 T1216 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 426792 T1228 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 426793 T1228 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 426796 T1244 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 426797 T1228 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 426798 T1244 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4f5204ca lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@40131d31),segFN=segments_1,generation=1}
   [junit4]   2> 426798 T1244 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 426799 T1244 oass.SolrIndexSearcher.<init> Opening Searcher@400ea298[solrj_collection_shard1_replica2] main
   [junit4]   2> 426801 T1216 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 426802 T1268 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica2] Registered new searcher Searcher@400ea298[solrj_collection_shard1_replica2] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 426804 T1216 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 426805 T1216 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 426805 T1216 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 426793 T1196 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 426807 T1196 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 426805 T1228 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@37f13bed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@14c6731),segFN=segments_1,generation=1}
   [junit4]   2> 426808 T1216 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 426805 T1244 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica2
   [junit4]   2> 426808 T1216 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 426808 T1196 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 426810 T1196 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5868a774 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@606f3b45),segFN=segments_1,generation=1}
   [junit4]   2> 426811 T1196 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 426811 T1196 oass.SolrIndexSearcher.<init> Opening Searcher@45e24db0[solrj_collection_shard2_replica2] main
   [junit4]   2> 426808 T1228 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 426809 T1216 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 426809 T1244 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica2 address:http://127.0.0.1:48095 collection:solrj_collection shard:shard1
   [junit4]   2> 426815 T1216 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/solrj_collection_shard2_replica1/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/solrj_collection_shard2_replica1/data/
   [junit4]   2> 426815 T1216 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@bc10d02
   [junit4]   2> 426816 T1270 oasc.SolrCore.registerSearcher [solrj_collection_shard2_replica2] Registered new searcher Searcher@45e24db0[solrj_collection_shard2_replica2] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 426817 T1216 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/solrj_collection_shard2_replica1/data
   [junit4]   2> 426817 T1216 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/solrj_collection_shard2_replica1/data/index/
   [junit4]   2> 426814 T1228 oass.SolrIndexSearcher.<init> Opening Searcher@640676af[solrj_collection_shard1_replica1] main
   [junit4]   2> 426818 T1216 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 426819 T1244 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
   [junit4]   2> 426819 T1216 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1392653916819/solrj_collection_shard2_replica1/data/index
   [junit4]   2> 426820 T1216 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=26, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=1.8603515625, floorSegmentMB=1.1904296875, forceMergeDeletesPctAllowed=17.8928489670199, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7490457491565093
   [junit4]   2> 426821 T1216 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@52e8a6cb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@138a91bf),segFN=segments_1,generation=1}
   [junit4]   2> 426821 T1216 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 426826 T1196 oasc.CoreContainer.registerCore registering core: solrj_collection_shard2_replica2
   [junit4]   2> 426826 T1216 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 426826 T1196 oasc.ZkController.register Register replica - core:solrj_collection_s

[...truncated too long message...]

rg.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 806386 T2403 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty11/index
   [junit4]   2> 806387 T2403 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty11 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty11;done=false>>]
   [junit4]   2> 806387 T2403 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty11
   [junit4]   2> 806386 T1838 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 806388 T2403 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 806388 T2403 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 806386 T2097 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty10/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty10/index;done=false>>]
   [junit4]   2> 806389 T2097 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty10/index
   [junit4]   2> 806388 T1838 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:254)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 806389 T2097 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty10 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty10;done=false>>]
   [junit4]   2> 806390 T2097 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1392654162229/jetty10
   [junit4]   2> 806390 T2097 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene42, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=et_EE, timezone=US/Pacific-New
   [junit4]   2> NOTE: Linux 3.8.0-35-generic amd64/Sun Microsystems Inc. 1.6.0_45 (64-bit)/cpus=8,threads=1,free=123346288,total=376823808
   [junit4]   2> NOTE: All tests run in this JVM: [MultiTermTest, XmlUpdateRequestHandlerTest, TestRandomMergePolicy, HighlighterConfigTest, TestSchemaResource, TestStressReorder, RequiredFieldsTest, DistributedQueryComponentCustomSortTest, SimplePostToolTest, TestOmitPositions, TestFastWriter, TestMultiCoreConfBootstrap, HdfsBasicDistributedZk2Test, DistributedSuggestComponentTest, TestFoldingMultitermQuery, TestSolrXmlPersistence, NotRequiredUniqueKeyTest, DefaultValueUpdateProcessorTest, OpenCloseCoreStressTest, TestDistribDocBasedVersion, TestUtils, TestQueryTypes, DateFieldTest, DirectUpdateHandlerTest, TestShardHandlerFactory, HdfsUnloadDistributedZkTest, CoreContainerCoreInitFailuresTest, TestDFRSimilarityFactory, TestCSVResponseWriter, TestRTGBase, DocValuesMultiTest, SpellCheckComponentTest, TestMaxScoreQueryParser, TestReloadAndDeleteDocs, TestBinaryField, TestCodecSupport, TestStressUserVersions, BadCopyFieldTest, FieldAnalysisRequestHandlerTest, TestDistributedGrouping, TestXIncludeConfig, PluginInfoTest, TestNumberUtils, TestZkChroot, IndexBasedSpellCheckerTest, TestSolr4Spatial, CurrencyFieldOpenExchangeTest, SpellingQueryConverterTest, TestFastOutputStream, TestSimpleQParserPlugin, TestComponentsName, LeaderElectionIntegrationTest, TestHighlightDedupGrouping, CursorPagingTest, TestRandomFaceting, TestSchemaNameResource, CollectionsAPIDistributedZkTest, TimeZoneUtilsTest, TestCollationFieldDocValues, TestClassNameShortening, ShardRoutingCustomTest, TermVectorComponentDistributedTest, XsltUpdateRequestHandlerTest, TestBinaryResponseWriter, SignatureUpdateProcessorFactoryTest, NumericFieldsTest, TestRecoveryHdfs, TestImplicitCoreProperties, ConnectionManagerTest, TestStressLucene, TestFieldTypeResource, DebugComponentTest, StandardRequestHandlerTest, AssignTest, TestIndexingPerformance, TestCollationField, ChangedSchemaMergeTest, TestCoreDiscovery, TestQuerySenderListener, TestUpdate, ZkNodePropsTest, ChaosMonkeySafeLeaderTest]
   [junit4] Completed on J1 in 142.83s, 1 test, 1 error <<< FAILURES!

[...truncated 607 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:459: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:439: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:491: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1308: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:941: There were test failures: 362 suites, 1563 tests, 2 errors, 36 ignored (8 assumptions)

Total time: 51 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.6.0_45 -XX:+UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure