You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2013/05/29 01:23:16 UTC
[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 270 - Failure
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/270/
1 tests failed.
REGRESSION: org.apache.solr.cloud.SyncSliceTest.testDistribSearch
Error Message:
shard1 is not consistent. Got 305 from http://127.0.0.1:41961/sg_/collection1lastClient and got 5 from http://127.0.0.1:13456/sg_/collection1
Stack Trace:
java.lang.AssertionError: shard1 is not consistent. Got 305 from http://127.0.0.1:41961/sg_/collection1lastClient and got 5 from http://127.0.0.1:13456/sg_/collection1
at __randomizedtesting.SeedInfo.seed([5629969592A39EB0:D7CF188DE5FCFE8C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:963)
at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:238)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:679)
Build Log:
[...truncated 9285 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.SyncSliceTest
[junit4:junit4] 2> 1475150 T2999 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /sg_/
[junit4:junit4] 2> 1475155 T2999 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./solrtest-SyncSliceTest-1369781902029
[junit4:junit4] 2> 1475156 T2999 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 1475157 T3000 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 1475258 T2999 oasc.ZkTestServer.run start zk server on port:63187
[junit4:junit4] 2> 1475259 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1475263 T3006 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77bbaa10 name:ZooKeeperConnection Watcher:127.0.0.1:63187 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1475263 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1475264 T2999 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 1475284 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1475286 T3008 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b1c313c name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1475287 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1475287 T2999 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 1475298 T2999 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 1475301 T2999 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 1475304 T2999 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 1475312 T2999 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1475313 T2999 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 1475327 T2999 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 1475327 T2999 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 1475442 T2999 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1475443 T2999 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 1475446 T2999 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 1475447 T2999 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 1475450 T2999 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 1475451 T2999 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 1475454 T2999 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1475455 T2999 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 1475457 T2999 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1475458 T2999 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 1475461 T2999 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1475462 T2999 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 1475469 T2999 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1475470 T2999 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 1475720 T2999 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1475723 T2999 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:49557
[junit4:junit4] 2> 1475723 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1475724 T2999 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1475724 T2999 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-controljetty-1369781902346
[junit4:junit4] 2> 1475725 T2999 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-controljetty-1369781902346/solr.xml
[junit4:junit4] 2> 1475725 T2999 oasc.CoreContainer.<init> New CoreContainer 1196032049
[junit4:junit4] 2> 1475726 T2999 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-controljetty-1369781902346/'
[junit4:junit4] 2> 1475726 T2999 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-controljetty-1369781902346/'
[junit4:junit4] 2> 1475840 T2999 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1475841 T2999 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1475842 T2999 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1475842 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1475843 T2999 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1475843 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1475843 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1475844 T2999 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1475844 T2999 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1475845 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1475861 T2999 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1475862 T2999 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63187/solr
[junit4:junit4] 2> 1475863 T2999 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1475864 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1475893 T3019 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22bbe1a4 name:ZooKeeperConnection Watcher:127.0.0.1:63187 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1475893 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1475903 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1475915 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1475917 T3021 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73dd6307 name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1475917 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1475919 T2999 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 1475929 T2999 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 1475933 T2999 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 1475939 T2999 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49557_sg_
[junit4:junit4] 2> 1475941 T2999 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49557_sg_
[junit4:junit4] 2> 1475944 T2999 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 1475963 T2999 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 1475966 T2999 oasc.Overseer.start Overseer (id=89770026731831299-127.0.0.1:49557_sg_-n_0000000000) starting
[junit4:junit4] 2> 1475979 T2999 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 1475991 T3023 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 1475991 T2999 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 1475994 T2999 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 1475996 T2999 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1475998 T3022 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 1476002 T3024 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-controljetty-1369781902346/collection1
[junit4:junit4] 2> 1476002 T3024 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 1476003 T3024 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1476003 T3024 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 1476004 T3024 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-controljetty-1369781902346/collection1/'
[junit4:junit4] 2> 1476006 T3024 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-controljetty-1369781902346/collection1/lib/README' to classloader
[junit4:junit4] 2> 1476006 T3024 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-controljetty-1369781902346/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1476056 T3024 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 1476114 T3024 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1476215 T3024 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1476222 T3024 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1476831 T3024 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1476844 T3024 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1476848 T3024 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1476861 T3024 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1476866 T3024 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1476870 T3024 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1476871 T3024 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1476872 T3024 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1476872 T3024 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1476873 T3024 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1476873 T3024 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1476873 T3024 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1476920 T3024 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-controljetty-1369781902346/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1369781902028/control/data/
[junit4:junit4] 2> 1476920 T3024 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@26dfe303
[junit4:junit4] 2> 1476921 T3024 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1476922 T3024 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/control/data
[junit4:junit4] 2> 1476923 T3024 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1369781902028/control/data/index/
[junit4:junit4] 2> 1476924 T3024 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1369781902028/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1476925 T3024 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/control/data/index
[junit4:junit4] 2> 1476931 T3024 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/control/data/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1476931 T3024 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1476934 T3024 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1476934 T3024 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1476935 T3024 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1476935 T3024 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1476936 T3024 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1476936 T3024 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1476936 T3024 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1476937 T3024 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1476937 T3024 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1476947 T3024 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1476956 T3024 oass.SolrIndexSearcher.<init> Opening Searcher@49e397b main
[junit4:junit4] 2> 1476957 T3024 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1476958 T3024 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1476961 T3025 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@49e397b main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1476963 T3024 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1476963 T3024 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1477501 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1477502 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49557_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49557/sg_"}
[junit4:junit4] 2> 1477502 T3022 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 1477503 T3022 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1477516 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1477966 T3024 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1477966 T3024 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49557/sg_ collection:control_collection shard:shard1
[junit4:junit4] 2> 1477967 T3024 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 1477984 T3024 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1477986 T3024 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1477987 T3024 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1477987 T3024 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49557/sg_/collection1/
[junit4:junit4] 2> 1477987 T3024 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1477987 T3024 oasc.SyncStrategy.syncToMe http://127.0.0.1:49557/sg_/collection1/ has no replicas
[junit4:junit4] 2> 1477988 T3024 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49557/sg_/collection1/
[junit4:junit4] 2> 1477988 T3024 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 1479019 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1479111 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1479182 T3024 oasc.ZkController.register We are http://127.0.0.1:49557/sg_/collection1/ and leader is http://127.0.0.1:49557/sg_/collection1/
[junit4:junit4] 2> 1479182 T3024 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49557/sg_
[junit4:junit4] 2> 1479182 T3024 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1479182 T3024 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1479183 T3024 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1479185 T3024 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1479186 T2999 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1479186 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1479187 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1479202 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 1479203 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1479211 T3028 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26dab4e5 name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1479212 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1479213 T2999 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1479215 T2999 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 1479468 T2999 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1479471 T2999 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:64652
[junit4:junit4] 2> 1479471 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1479472 T2999 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1479472 T2999 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty1-1369781906088
[junit4:junit4] 2> 1479473 T2999 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty1-1369781906088/solr.xml
[junit4:junit4] 2> 1479473 T2999 oasc.CoreContainer.<init> New CoreContainer 1034263606
[junit4:junit4] 2> 1479474 T2999 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty1-1369781906088/'
[junit4:junit4] 2> 1479474 T2999 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty1-1369781906088/'
[junit4:junit4] 2> 1479585 T2999 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1479586 T2999 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1479586 T2999 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1479587 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1479587 T2999 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1479588 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1479588 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1479589 T2999 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1479589 T2999 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1479590 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1479605 T2999 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1479605 T2999 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63187/solr
[junit4:junit4] 2> 1479606 T2999 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1479607 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1479613 T3039 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b8c2ba1 name:ZooKeeperConnection Watcher:127.0.0.1:63187 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1479614 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1479628 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1479639 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1479641 T3041 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@495d17d1 name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1479641 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1479652 T2999 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1480655 T2999 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:64652_sg_
[junit4:junit4] 2> 1480657 T2999 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:64652_sg_
[junit4:junit4] 2> 1480658 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1480661 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:49557_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:49557/sg_"}
[junit4:junit4] 2> 1480661 T3028 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1480661 T3041 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1480679 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1480679 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1480683 T3042 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty1-1369781906088/collection1
[junit4:junit4] 2> 1480683 T3042 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1480688 T3042 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1480688 T3042 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1480689 T3021 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 1480689 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1480690 T3042 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty1-1369781906088/collection1/'
[junit4:junit4] 2> 1480690 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1480691 T3042 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty1-1369781906088/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1480692 T3042 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty1-1369781906088/collection1/lib/README' to classloader
[junit4:junit4] 2> 1480741 T3042 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 1480798 T3042 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1480899 T3042 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1480906 T3042 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1481513 T3042 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1481525 T3042 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1481530 T3042 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1481543 T3042 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1481548 T3042 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1481552 T3042 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1481553 T3042 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1481553 T3042 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1481554 T3042 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1481555 T3042 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1481555 T3042 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1481555 T3042 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1481555 T3042 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty1-1369781906088/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/
[junit4:junit4] 2> 1481556 T3042 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@26dfe303
[junit4:junit4] 2> 1481556 T3042 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1481557 T3042 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1
[junit4:junit4] 2> 1481558 T3042 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index/
[junit4:junit4] 2> 1481559 T3042 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1481559 T3042 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index
[junit4:junit4] 2> 1481565 T3042 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1481565 T3042 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1481568 T3042 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1481568 T3042 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1481568 T3042 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1481569 T3042 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1481570 T3042 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1481570 T3042 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1481570 T3042 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1481571 T3042 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1481571 T3042 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1481580 T3042 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1481588 T3042 oass.SolrIndexSearcher.<init> Opening Searcher@35fb2821 main
[junit4:junit4] 2> 1481589 T3042 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1481590 T3042 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1481594 T3043 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@35fb2821 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1481596 T3042 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1481596 T3042 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1482190 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1482191 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:64652_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:64652/sg_"}
[junit4:junit4] 2> 1482191 T3022 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 1
[junit4:junit4] 2> 1482192 T3022 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1482196 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1482196 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1482196 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1482599 T3042 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1482599 T3042 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:64652/sg_ collection:collection1 shard:shard1
[junit4:junit4] 2> 1482600 T3042 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 1482618 T3042 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 1482627 T3042 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 1482627 T3042 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 1482627 T3042 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:64652/sg_/collection1/
[junit4:junit4] 2> 1482627 T3042 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 1482628 T3042 oasc.SyncStrategy.syncToMe http://127.0.0.1:64652/sg_/collection1/ has no replicas
[junit4:junit4] 2> 1482628 T3042 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:64652/sg_/collection1/
[junit4:junit4] 2> 1482628 T3042 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 1483713 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1483731 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1483731 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1483731 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1483763 T3042 oasc.ZkController.register We are http://127.0.0.1:64652/sg_/collection1/ and leader is http://127.0.0.1:64652/sg_/collection1/
[junit4:junit4] 2> 1483763 T3042 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:64652/sg_
[junit4:junit4] 2> 1483763 T3042 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 1483763 T3042 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1483764 T3042 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1483767 T3042 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1483768 T2999 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1483768 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1483769 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1484035 T2999 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1484038 T2999 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62500
[junit4:junit4] 2> 1484039 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1484039 T2999 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1484040 T2999 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty2-1369781910657
[junit4:junit4] 2> 1484040 T2999 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty2-1369781910657/solr.xml
[junit4:junit4] 2> 1484041 T2999 oasc.CoreContainer.<init> New CoreContainer 950561580
[junit4:junit4] 2> 1484041 T2999 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty2-1369781910657/'
[junit4:junit4] 2> 1484042 T2999 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty2-1369781910657/'
[junit4:junit4] 2> 1484151 T2999 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1484152 T2999 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1484152 T2999 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1484153 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1484153 T2999 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1484154 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1484154 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1484155 T2999 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1484155 T2999 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1484156 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1484171 T2999 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1484171 T2999 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63187/solr
[junit4:junit4] 2> 1484172 T2999 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1484173 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1484222 T3055 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d10bd7b name:ZooKeeperConnection Watcher:127.0.0.1:63187 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1484223 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1484268 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1484278 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1484282 T3057 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b243ba4 name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1484282 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1484344 T2999 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1485236 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1485237 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:64652_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:64652/sg_"}
[junit4:junit4] 2> 1485255 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1485255 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1485255 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1485256 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1485347 T2999 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62500_sg_
[junit4:junit4] 2> 1485349 T2999 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62500_sg_
[junit4:junit4] 2> 1485351 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1485351 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1485352 T3021 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1485352 T3028 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1485352 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1485352 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1485353 T3057 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1485363 T3041 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 1485368 T3058 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty2-1369781910657/collection1
[junit4:junit4] 2> 1485369 T3058 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1485369 T3058 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1485369 T3058 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1485371 T3058 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty2-1369781910657/collection1/'
[junit4:junit4] 2> 1485372 T3058 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty2-1369781910657/collection1/lib/README' to classloader
[junit4:junit4] 2> 1485373 T3058 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty2-1369781910657/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1485422 T3058 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 1485483 T3058 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1485584 T3058 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1485591 T3058 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1486193 T3058 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1486205 T3058 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1486209 T3058 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1486223 T3058 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1486227 T3058 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1486235 T3058 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1486236 T3058 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1486237 T3058 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1486237 T3058 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1486238 T3058 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1486238 T3058 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1486239 T3058 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1486239 T3058 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty2-1369781910657/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/
[junit4:junit4] 2> 1486239 T3058 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@26dfe303
[junit4:junit4] 2> 1486240 T3058 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1486241 T3058 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2
[junit4:junit4] 2> 1486242 T3058 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index/
[junit4:junit4] 2> 1486243 T3058 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1486243 T3058 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index
[junit4:junit4] 2> 1486249 T3058 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1486249 T3058 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1486252 T3058 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1486252 T3058 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1486253 T3058 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1486253 T3058 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1486254 T3058 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1486254 T3058 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1486255 T3058 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1486255 T3058 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1486256 T3058 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1486265 T3058 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1486274 T3058 oass.SolrIndexSearcher.<init> Opening Searcher@469b6538 main
[junit4:junit4] 2> 1486276 T3058 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1486276 T3058 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1486280 T3059 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@469b6538 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1486283 T3058 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1486283 T3058 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1486760 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1486761 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:62500_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:62500/sg_"}
[junit4:junit4] 2> 1486761 T3022 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
[junit4:junit4] 2> 1486762 T3022 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1486780 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1486780 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1486780 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1486780 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1487286 T3058 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1487286 T3058 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62500/sg_ collection:collection1 shard:shard1
[junit4:junit4] 2> 1487289 T3058 oasc.ZkController.register We are http://127.0.0.1:62500/sg_/collection1/ and leader is http://127.0.0.1:64652/sg_/collection1/
[junit4:junit4] 2> 1487290 T3058 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62500/sg_
[junit4:junit4] 2> 1487290 T3058 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1487290 T3058 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C920 name=collection1 org.apache.solr.core.SolrCore@2bc4c404 url=http://127.0.0.1:62500/sg_/collection1 node=127.0.0.1:62500_sg_ C920_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:62500_sg_, base_url=http://127.0.0.1:62500/sg_}
[junit4:junit4] 2> 1487291 T3060 C920 P62500 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1487291 T3058 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1487292 T3060 C920 P62500 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1487292 T3060 C920 P62500 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1487292 T3060 C920 P62500 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1487293 T2999 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1487293 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1487294 T3060 C920 P62500 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1487294 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1487317 T3031 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 3, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1487574 T2999 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1487577 T2999 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:21322
[junit4:junit4] 2> 1487577 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1487578 T2999 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1487578 T2999 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty3-1369781914189
[junit4:junit4] 2> 1487579 T2999 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty3-1369781914189/solr.xml
[junit4:junit4] 2> 1487579 T2999 oasc.CoreContainer.<init> New CoreContainer 986581466
[junit4:junit4] 2> 1487580 T2999 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty3-1369781914189/'
[junit4:junit4] 2> 1487580 T2999 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty3-1369781914189/'
[junit4:junit4] 2> 1487691 T2999 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1487691 T2999 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1487692 T2999 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1487692 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1487693 T2999 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1487693 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1487694 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1487694 T2999 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1487695 T2999 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1487695 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1487710 T2999 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1487711 T2999 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63187/solr
[junit4:junit4] 2> 1487711 T2999 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1487712 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1487715 T3072 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e2ad701 name:ZooKeeperConnection Watcher:127.0.0.1:63187 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1487716 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1487726 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1487736 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1487738 T3074 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69788ab9 name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1487738 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1487748 T2999 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1488284 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1488285 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:62500_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:62500/sg_"}
[junit4:junit4] 2> 1488305 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1488305 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1488305 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1488305 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1488305 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1488319 T3031 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 3, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1488319 T3031 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=3&state=recovering&nodeName=127.0.0.1:62500_sg_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1002
[junit4:junit4] 2> 1488751 T2999 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:21322_sg_
[junit4:junit4] 2> 1488753 T2999 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:21322_sg_
[junit4:junit4] 2> 1488755 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1488755 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1488755 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 1488755 T3021 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1488755 T3028 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1488756 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1488756 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1488757 T3057 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1488764 T3041 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1488764 T3074 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 1488769 T3075 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty3-1369781914189/collection1
[junit4:junit4] 2> 1488769 T3075 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1488770 T3075 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1488770 T3075 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1488771 T3075 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty3-1369781914189/collection1/'
[junit4:junit4] 2> 1488773 T3075 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty3-1369781914189/collection1/lib/README' to classloader
[junit4:junit4] 2> 1488773 T3075 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty3-1369781914189/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1488822 T3075 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 1488879 T3075 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1488980 T3075 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1488987 T3075 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1489609 T3075 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1489622 T3075 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1489626 T3075 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1489639 T3075 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1489644 T3075 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1489649 T3075 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1489649 T3075 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1489650 T3075 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1489650 T3075 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1489651 T3075 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1489652 T3075 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1489652 T3075 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1489652 T3075 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty3-1369781914189/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/
[junit4:junit4] 2> 1489652 T3075 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@26dfe303
[junit4:junit4] 2> 1489653 T3075 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1489654 T3075 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3
[junit4:junit4] 2> 1489655 T3075 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index/
[junit4:junit4] 2> 1489656 T3075 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1489657 T3075 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index
[junit4:junit4] 2> 1489662 T3075 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1489662 T3075 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1489665 T3075 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1489665 T3075 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1489666 T3075 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1489667 T3075 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1489667 T3075 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1489667 T3075 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1489668 T3075 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1489668 T3075 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1489669 T3075 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1489679 T3075 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1489689 T3075 oass.SolrIndexSearcher.<init> Opening Searcher@29a356ca main
[junit4:junit4] 2> 1489690 T3075 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1489691 T3075 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1489695 T3076 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@29a356ca main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1489696 T3075 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1489697 T3075 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1489809 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1489810 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:21322_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:21322/sg_"}
[junit4:junit4] 2> 1489810 T3022 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
[junit4:junit4] 2> 1489810 T3022 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1489829 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1489829 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1489829 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1489829 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1489829 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> ASYNC NEW_CORE C921 name=collection1 org.apache.solr.core.SolrCore@2bc4c404 url=http://127.0.0.1:62500/sg_/collection1 node=127.0.0.1:62500_sg_ C921_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:62500_sg_, base_url=http://127.0.0.1:62500/sg_}
[junit4:junit4] 2> 1490321 T3060 C921 P62500 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:64652/sg_/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1490321 T3060 C921 P62500 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:62500/sg_ START replicas=[http://127.0.0.1:64652/sg_/collection1/] nUpdates=100
[junit4:junit4] 2> 1490322 T3060 C921 P62500 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1490322 T3060 C921 P62500 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1490322 T3060 C921 P62500 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1490323 T3060 C921 P62500 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1490323 T3060 C921 P62500 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1490323 T3060 C921 P62500 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:64652/sg_/collection1/. core=collection1
[junit4:junit4] 2> 1490323 T3060 C921 P62500 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C922 name=collection1 org.apache.solr.core.SolrCore@711e7e71 url=http://127.0.0.1:64652/sg_/collection1 node=127.0.0.1:64652_sg_ C922_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:64652_sg_, base_url=http://127.0.0.1:64652/sg_, leader=true}
[junit4:junit4] 2> 1490328 T3032 C922 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 1490338 T3033 C922 P64652 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1490341 T3033 C922 P64652 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1490342 T3033 C922 P64652 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1490344 T3033 C922 P64652 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1490344 T3033 C922 P64652 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1490346 T3033 C922 P64652 oass.SolrIndexSearcher.<init> Opening Searcher@1d153c77 realtime
[junit4:junit4] 2> 1490346 T3033 C922 P64652 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1490346 T3033 C922 P64652 oasup.LogUpdateProcessor.finish [collection1] webapp=/sg_ path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 8
[junit4:junit4] 2> 1490347 T3060 C921 P62500 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1490347 T3060 C921 P62500 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1490348 T3034 C922 P64652 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1490349 T3034 C922 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=1
[junit4:junit4] 2> 1490349 T3060 C921 P62500 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 1490349 T3060 C921 P62500 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1490349 T3060 C921 P62500 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1490352 T3034 C922 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 1490353 T3060 C921 P62500 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1490354 T3060 C921 P62500 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index.20130529015837226
[junit4:junit4] 2> 1490355 T3060 C921 P62500 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index.20130529015837226 lockFactory=org.apache.lucene.store.NativeFSLockFactory@433cc6e6 fullCopy=false
[junit4:junit4] 2> 1490358 T3034 C922 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 1490359 T3060 C921 P62500 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 1490361 T3060 C921 P62500 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 1490361 T3060 C921 P62500 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 1490364 T3060 C921 P62500 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> 1490364 T3060 C921 P62500 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 1490364 T3060 C921 P62500 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 1490365 T3060 C921 P62500 oass.SolrIndexSearcher.<init> Opening Searcher@64e3d21f main
[junit4:junit4] 2> 1490366 T3059 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@64e3d21f main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 1490366 T3060 C921 P62500 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index.20130529015837226 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index.20130529015837226;done=true>>]
[junit4:junit4] 2> 1490366 T3060 C921 P62500 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index.20130529015837226
[junit4:junit4] 2> 1490367 T3060 C921 P62500 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty2/index.20130529015837226
[junit4:junit4] 2> 1490367 T3060 C921 P62500 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 1490367 T3060 C921 P62500 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 1490368 T3060 C921 P62500 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1490368 T3060 C921 P62500 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1490369 T3060 C921 P62500 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 1490701 T3075 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1490701 T3075 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:21322/sg_ collection:collection1 shard:shard1
[junit4:junit4] 2> 1490704 T3075 oasc.ZkController.register We are http://127.0.0.1:21322/sg_/collection1/ and leader is http://127.0.0.1:64652/sg_/collection1/
[junit4:junit4] 2> 1490704 T3075 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:21322/sg_
[junit4:junit4] 2> 1490704 T3075 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1490704 T3075 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C923 name=collection1 org.apache.solr.core.SolrCore@7dfdec4d url=http://127.0.0.1:21322/sg_/collection1 node=127.0.0.1:21322_sg_ C923_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:21322_sg_, base_url=http://127.0.0.1:21322/sg_}
[junit4:junit4] 2> 1490705 T3079 C923 P21322 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1490705 T3079 C923 P21322 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1490705 T3075 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1490705 T3079 C923 P21322 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1490706 T3079 C923 P21322 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1490706 T2999 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1490707 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1490707 T3079 C923 P21322 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1490707 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1490720 T3035 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1490879 T2999 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1490881 T2999 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51150
[junit4:junit4] 2> 1490882 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1490882 T2999 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1490883 T2999 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty4-1369781917592
[junit4:junit4] 2> 1490883 T2999 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty4-1369781917592/solr.xml
[junit4:junit4] 2> 1490883 T2999 oasc.CoreContainer.<init> New CoreContainer 626070847
[junit4:junit4] 2> 1490884 T2999 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty4-1369781917592/'
[junit4:junit4] 2> 1490884 T2999 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty4-1369781917592/'
[junit4:junit4] 2> 1490958 T2999 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1490959 T2999 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1490959 T2999 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1490960 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1490960 T2999 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1490960 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1490961 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1490961 T2999 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1490961 T2999 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1490961 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1490972 T2999 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1490972 T2999 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63187/solr
[junit4:junit4] 2> 1490973 T2999 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1490973 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1490976 T3091 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a7c58f0 name:ZooKeeperConnection Watcher:127.0.0.1:63187 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1490977 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1490989 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1490996 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1490999 T3093 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60ad3469 name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1490999 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1491011 T2999 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1491333 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1491334 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:62500_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:62500/sg_"}
[junit4:junit4] 2> 1491338 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:21322_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:21322/sg_"}
[junit4:junit4] 2> 1491356 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1491356 T3093 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1491356 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1491356 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1491357 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1491356 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1491722 T3035 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1491722 T3035 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=4&state=recovering&nodeName=127.0.0.1:21322_sg_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1002
[junit4:junit4] 2> 1492014 T2999 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51150_sg_
[junit4:junit4] 2> 1492059 T2999 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51150_sg_
[junit4:junit4] 2> 1492061 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1492061 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1492061 T3028 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1492061 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 1492062 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1492062 T3021 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1492061 T3093 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1492062 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1492063 T3041 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1492063 T3074 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1492063 T3093 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1492073 T3057 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 1492078 T3094 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty4-1369781917592/collection1
[junit4:junit4] 2> 1492078 T3094 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1492079 T3094 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1492079 T3094 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1492080 T3094 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty4-1369781917592/collection1/'
[junit4:junit4] 2> 1492082 T3094 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty4-1369781917592/collection1/lib/README' to classloader
[junit4:junit4] 2> 1492082 T3094 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty4-1369781917592/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1492131 T3094 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 1492188 T3094 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1492289 T3094 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1492295 T3094 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1492900 T3094 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1492912 T3094 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1492917 T3094 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1492930 T3094 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1492934 T3094 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1492939 T3094 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1492940 T3094 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1492940 T3094 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1492940 T3094 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1492941 T3094 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1492941 T3094 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1492942 T3094 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1492942 T3094 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty4-1369781917592/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/
[junit4:junit4] 2> 1492942 T3094 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@26dfe303
[junit4:junit4] 2> 1492943 T3094 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1492944 T3094 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4
[junit4:junit4] 2> 1492945 T3094 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index/
[junit4:junit4] 2> 1492946 T3094 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1492946 T3094 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index
[junit4:junit4] 2> 1492952 T3094 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1492952 T3094 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1492955 T3094 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1492955 T3094 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1492956 T3094 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1492957 T3094 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1492957 T3094 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1492957 T3094 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1492958 T3094 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1492958 T3094 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1492959 T3094 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1492968 T3094 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1492978 T3094 oass.SolrIndexSearcher.<init> Opening Searcher@6119765a main
[junit4:junit4] 2> 1492979 T3094 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1492979 T3094 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1492984 T3095 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6119765a main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1492986 T3094 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1492986 T3094 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> ASYNC NEW_CORE C924 name=collection1 org.apache.solr.core.SolrCore@7dfdec4d url=http://127.0.0.1:21322/sg_/collection1 node=127.0.0.1:21322_sg_ C924_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:21322_sg_, base_url=http://127.0.0.1:21322/sg_}
[junit4:junit4] 2> 1493724 T3079 C924 P21322 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:64652/sg_/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1493724 T3079 C924 P21322 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:21322/sg_ START replicas=[http://127.0.0.1:64652/sg_/collection1/] nUpdates=100
[junit4:junit4] 2> 1493725 T3079 C924 P21322 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1493725 T3079 C924 P21322 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1493725 T3079 C924 P21322 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1493726 T3079 C924 P21322 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1493726 T3079 C924 P21322 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1493726 T3079 C924 P21322 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:64652/sg_/collection1/. core=collection1
[junit4:junit4] 2> 1493726 T3079 C924 P21322 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C925 name=collection1 org.apache.solr.core.SolrCore@711e7e71 url=http://127.0.0.1:64652/sg_/collection1 node=127.0.0.1:64652_sg_ C925_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:64652_sg_, base_url=http://127.0.0.1:64652/sg_, leader=true}
[junit4:junit4] 2> 1493727 T3032 C925 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 1493744 T3036 C925 P64652 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1493748 T3036 C925 P64652 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_3,generation=3}
[junit4:junit4] 2> 1493748 T3036 C925 P64652 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 1493750 T3036 C925 P64652 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1493750 T3036 C925 P64652 oasup.LogUpdateProcessor.finish [collection1] webapp=/sg_ path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 6
[junit4:junit4] 2> 1493751 T3079 C924 P21322 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1493751 T3079 C924 P21322 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1493753 T3034 C925 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=0
[junit4:junit4] 2> 1493754 T3079 C924 P21322 oash.SnapPuller.fetchLatestIndex Master's generation: 3
[junit4:junit4] 2> 1493754 T3079 C924 P21322 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1493754 T3079 C924 P21322 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1493757 T3034 C925 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=3&version=2} status=0 QTime=1
[junit4:junit4] 2> 1493757 T3079 C924 P21322 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1493759 T3079 C924 P21322 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index.20130529015840631
[junit4:junit4] 2> 1493760 T3079 C924 P21322 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index.20130529015840631 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1b8406cd fullCopy=false
[junit4:junit4] 2> 1493763 T3034 C925 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/replication params={file=segments_3&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=3} status=0 QTime=0
[junit4:junit4] 2> 1493764 T3079 C924 P21322 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 1493767 T3079 C924 P21322 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 1493767 T3079 C924 P21322 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 1493771 T3079 C924 P21322 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index,segFN=segments_3,generation=3}
[junit4:junit4] 2> 1493772 T3079 C924 P21322 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 1493772 T3079 C924 P21322 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 1493773 T3079 C924 P21322 oass.SolrIndexSearcher.<init> Opening Searcher@29cdd2b4 main
[junit4:junit4] 2> 1493774 T3076 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@29cdd2b4 main{StandardDirectoryReader(segments_3:1:nrt)}
[junit4:junit4] 2> 1493775 T3079 C924 P21322 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index.20130529015840631 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index.20130529015840631;done=true>>]
[junit4:junit4] 2> 1493775 T3079 C924 P21322 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index.20130529015840631
[junit4:junit4] 2> 1493775 T3079 C924 P21322 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty3/index.20130529015840631
[junit4:junit4] 2> 1493776 T3079 C924 P21322 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 1493776 T3079 C924 P21322 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 1493777 T3079 C924 P21322 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1493777 T3079 C924 P21322 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1493778 T3079 C924 P21322 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 1494371 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1494372 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:51150_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:51150/sg_"}
[junit4:junit4] 2> 1494372 T3022 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
[junit4:junit4] 2> 1494373 T3022 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1494378 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:21322_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:21322/sg_"}
[junit4:junit4] 2> 1494402 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1494402 T3093 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1494402 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1494402 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1494402 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1494402 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1494990 T3094 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1494990 T3094 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51150/sg_ collection:collection1 shard:shard1
[junit4:junit4] 2> 1495020 T3094 oasc.ZkController.register We are http://127.0.0.1:51150/sg_/collection1/ and leader is http://127.0.0.1:64652/sg_/collection1/
[junit4:junit4] 2> 1495020 T3094 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51150/sg_
[junit4:junit4] 2> 1495021 T3094 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1495021 T3094 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C926 name=collection1 org.apache.solr.core.SolrCore@271806f7 url=http://127.0.0.1:51150/sg_/collection1 node=127.0.0.1:51150_sg_ C926_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:51150_sg_, base_url=http://127.0.0.1:51150/sg_}
[junit4:junit4] 2> 1495022 T3098 C926 P51150 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1495022 T3098 C926 P51150 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1495022 T3094 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1495023 T3098 C926 P51150 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1495023 T3098 C926 P51150 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1495024 T2999 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1495024 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1495025 T3098 C926 P51150 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1495025 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1495049 T3037 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1495293 T2999 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1495296 T2999 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41961
[junit4:junit4] 2> 1495297 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1495297 T2999 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1495298 T2999 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty5-1369781921921
[junit4:junit4] 2> 1495298 T2999 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty5-1369781921921/solr.xml
[junit4:junit4] 2> 1495299 T2999 oasc.CoreContainer.<init> New CoreContainer 2004395641
[junit4:junit4] 2> 1495299 T2999 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty5-1369781921921/'
[junit4:junit4] 2> 1495300 T2999 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty5-1369781921921/'
[junit4:junit4] 2> 1495407 T2999 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1495408 T2999 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1495408 T2999 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1495409 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1495409 T2999 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1495410 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1495410 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1495411 T2999 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1495411 T2999 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1495412 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1495426 T2999 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1495427 T2999 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63187/solr
[junit4:junit4] 2> 1495427 T2999 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1495428 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1495443 T3110 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17292b72 name:ZooKeeperConnection Watcher:127.0.0.1:63187 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1495444 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1495458 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1495468 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1495470 T3112 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37b1f6bc name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1495470 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1495481 T2999 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1495908 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1495909 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:51150_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:51150/sg_"}
[junit4:junit4] 2> 1495922 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1495922 T3112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1495922 T3093 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1495922 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1495922 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1495922 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1495922 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1496050 T3037 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 1496050 T3037 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:51150_sg_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1001
[junit4:junit4] 2> 1496484 T2999 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41961_sg_
[junit4:junit4] 2> 1496486 T2999 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41961_sg_
[junit4:junit4] 2> 1496489 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1496489 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1496489 T3093 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1496489 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 1496489 T3093 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1496489 T3112 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1496489 T3021 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1496489 T3028 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1496490 T3074 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1496490 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1496490 T3112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1496491 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1496491 T3057 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1496497 T3041 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4:junit4] 2> 1496503 T3113 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty5-1369781921921/collection1
[junit4:junit4] 2> 1496503 T3113 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1496504 T3113 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1496504 T3113 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1496506 T3113 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty5-1369781921921/collection1/'
[junit4:junit4] 2> 1496507 T3113 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty5-1369781921921/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1496508 T3113 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty5-1369781921921/collection1/lib/README' to classloader
[junit4:junit4] 2> 1496568 T3113 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 1496633 T3113 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1496735 T3113 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1496742 T3113 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1497383 T3113 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1497395 T3113 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1497399 T3113 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1497412 T3113 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1497417 T3113 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1497421 T3113 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1497422 T3113 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1497422 T3113 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1497423 T3113 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1497424 T3113 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1497424 T3113 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1497424 T3113 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1497424 T3113 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty5-1369781921921/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty5/
[junit4:junit4] 2> 1497425 T3113 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@26dfe303
[junit4:junit4] 2> 1497425 T3113 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1497426 T3113 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty5
[junit4:junit4] 2> 1497427 T3113 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty5/index/
[junit4:junit4] 2> 1497428 T3113 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty5/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1497429 T3113 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty5/index
[junit4:junit4] 2> 1497434 T3113 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty5/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1497434 T3113 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1497437 T3113 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1497437 T3113 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1497438 T3113 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1497438 T3113 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1497439 T3113 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1497439 T3113 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1497440 T3113 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1497440 T3113 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1497441 T3113 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1497450 T3113 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1497459 T3113 oass.SolrIndexSearcher.<init> Opening Searcher@1d66290d main
[junit4:junit4] 2> 1497460 T3113 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1497461 T3113 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1497465 T3114 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1d66290d main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1497467 T3113 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1497467 T3113 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> ASYNC NEW_CORE C927 name=collection1 org.apache.solr.core.SolrCore@271806f7 url=http://127.0.0.1:51150/sg_/collection1 node=127.0.0.1:51150_sg_ C927_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:51150_sg_, base_url=http://127.0.0.1:51150/sg_}
[junit4:junit4] 2> 1498052 T3098 C927 P51150 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:64652/sg_/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1498052 T3098 C927 P51150 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:51150/sg_ START replicas=[http://127.0.0.1:64652/sg_/collection1/] nUpdates=100
[junit4:junit4] 2> 1498052 T3098 C927 P51150 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1498053 T3098 C927 P51150 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1498053 T3098 C927 P51150 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1498053 T3098 C927 P51150 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1498054 T3098 C927 P51150 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1498054 T3098 C927 P51150 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:64652/sg_/collection1/. core=collection1
[junit4:junit4] 2> 1498054 T3098 C927 P51150 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C928 name=collection1 org.apache.solr.core.SolrCore@711e7e71 url=http://127.0.0.1:64652/sg_/collection1 node=127.0.0.1:64652_sg_ C928_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:64652_sg_, base_url=http://127.0.0.1:64652/sg_, leader=true}
[junit4:junit4] 2> 1498055 T3032 C928 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 1498070 T3031 C928 P64652 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1498108 T3031 C928 P64652 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=3
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_3,generation=3}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_4,generation=4}
[junit4:junit4] 2> 1498108 T3031 C928 P64652 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
[junit4:junit4] 2> 1498109 T3031 C928 P64652 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1498110 T3031 C928 P64652 oasup.LogUpdateProcessor.finish [collection1] webapp=/sg_ path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 40
[junit4:junit4] 2> 1498111 T3098 C927 P51150 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1498111 T3098 C927 P51150 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1498113 T3034 C928 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=0
[junit4:junit4] 2> 1498114 T3098 C927 P51150 oash.SnapPuller.fetchLatestIndex Master's generation: 4
[junit4:junit4] 2> 1498114 T3098 C927 P51150 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 1498114 T3098 C927 P51150 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 1498117 T3034 C928 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=4&version=2} status=0 QTime=1
[junit4:junit4] 2> 1498117 T3098 C927 P51150 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 1498119 T3098 C927 P51150 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index.20130529015844991
[junit4:junit4] 2> 1498120 T3098 C927 P51150 oash.SnapPuller.fetchLatestIndex Starting download to org.apache.lucene.store.NIOFSDirectory@/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index.20130529015844991 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4d3af6ef fullCopy=false
[junit4:junit4] 2> 1498123 T3034 C928 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/replication params={file=segments_4&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=4} status=0 QTime=0
[junit4:junit4] 2> 1498124 T3098 C927 P51150 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 1498129 T3098 C927 P51150 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 1498129 T3098 C927 P51150 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 1498134 T3098 C927 P51150 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index,segFN=segments_4,generation=4}
[junit4:junit4] 2> 1498134 T3098 C927 P51150 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
[junit4:junit4] 2> 1498135 T3098 C927 P51150 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 1498136 T3098 C927 P51150 oass.SolrIndexSearcher.<init> Opening Searcher@475a90d8 main
[junit4:junit4] 2> 1498137 T3095 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@475a90d8 main{StandardDirectoryReader(segments_4:1:nrt)}
[junit4:junit4] 2> 1498137 T3098 C927 P51150 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index.20130529015844991 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index.20130529015844991;done=true>>]
[junit4:junit4] 2> 1498137 T3098 C927 P51150 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index.20130529015844991
[junit4:junit4] 2> 1498138 T3098 C927 P51150 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty4/index.20130529015844991
[junit4:junit4] 2> 1498139 T3098 C927 P51150 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 1498139 T3098 C927 P51150 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 1498139 T3098 C927 P51150 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 1498139 T3098 C927 P51150 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1498141 T3098 C927 P51150 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 1498929 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1498930 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"6",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:41961_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:41961/sg_"}
[junit4:junit4] 2> 1498930 T3022 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
[junit4:junit4] 2> 1498930 T3022 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1498934 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:51150_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:51150/sg_"}
[junit4:junit4] 2> 1498950 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1498951 T3112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1498950 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1498951 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1498951 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1498951 T3093 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1498953 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1499471 T3113 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1499471 T3113 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41961/sg_ collection:collection1 shard:shard1
[junit4:junit4] 2> 1499477 T3113 oasc.ZkController.register We are http://127.0.0.1:41961/sg_/collection1/ and leader is http://127.0.0.1:64652/sg_/collection1/
[junit4:junit4] 2> 1499478 T3113 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:41961/sg_
[junit4:junit4] 2> 1499478 T3113 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1499478 T3113 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C929 name=collection1 org.apache.solr.core.SolrCore@7044b240 url=http://127.0.0.1:41961/sg_/collection1 node=127.0.0.1:41961_sg_ C929_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:41961_sg_, base_url=http://127.0.0.1:41961/sg_}
[junit4:junit4] 2> 1499478 T3117 C929 P41961 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1499479 T3113 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1499479 T3117 C929 P41961 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1499480 T3117 C929 P41961 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1499480 T3117 C929 P41961 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1499480 T2999 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1499481 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1499481 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1499506 T3117 C929 P41961 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1499517 T3033 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 6, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1499742 T2999 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1499745 T2999 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:13456
[junit4:junit4] 2> 1499746 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1499746 T2999 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1499746 T2999 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty6-1369781926371
[junit4:junit4] 2> 1499747 T2999 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty6-1369781926371/solr.xml
[junit4:junit4] 2> 1499747 T2999 oasc.CoreContainer.<init> New CoreContainer 2085108718
[junit4:junit4] 2> 1499748 T2999 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty6-1369781926371/'
[junit4:junit4] 2> 1499748 T2999 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty6-1369781926371/'
[junit4:junit4] 2> 1499856 T2999 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1499857 T2999 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1499857 T2999 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1499858 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1499858 T2999 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1499859 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1499859 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1499860 T2999 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1499860 T2999 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1499861 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1499876 T2999 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1499876 T2999 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63187/solr
[junit4:junit4] 2> 1499877 T2999 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1499878 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1499881 T3129 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66e1608a name:ZooKeeperConnection Watcher:127.0.0.1:63187 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1499881 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1499891 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1499901 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1499903 T3131 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12967cd5 name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1499903 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1499913 T2999 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 1500508 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1500509 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"6",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:41961_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:41961/sg_"}
[junit4:junit4] 2> 1500522 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500522 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500522 T3112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500522 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500522 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500522 T3093 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500522 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500522 T3131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500916 T2999 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:13456_sg_
[junit4:junit4] 2> 1500918 T2999 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:13456_sg_
[junit4:junit4] 2> 1500920 T3131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500920 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500920 T3028 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1500921 T3112 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1500921 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1500921 T3093 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1500920 T3021 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1500920 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500920 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 6)
[junit4:junit4] 2> 1500922 T3041 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1500922 T3131 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1500922 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1500921 T3093 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1500921 T3112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1500930 T3074 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1500930 T3057 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4:junit4] 2> 1500934 T3132 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty6-1369781926371/collection1
[junit4:junit4] 2> 1500934 T3132 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 1500935 T3132 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 1500935 T3132 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 1500937 T3132 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty6-1369781926371/collection1/'
[junit4:junit4] 2> 1500938 T3132 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty6-1369781926371/collection1/lib/README' to classloader
[junit4:junit4] 2> 1500939 T3132 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-jetty6-1369781926371/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 1500992 T3132 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 1501050 T3132 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 1501052 T3132 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 1501058 T3132 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 1501519 T3033 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 6, state: recovering, checkLive: true, onlyIfLeader: true for: 2 seconds.
[junit4:junit4] 2> 1501519 T3033 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=6&state=recovering&nodeName=127.0.0.1:41961_sg_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=2002
[junit4:junit4] 2> 1501715 T3132 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
[junit4:junit4] 2> 1501729 T3132 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 1501733 T3132 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 1501748 T3132 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1501755 T3132 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 1501761 T3132 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1501762 T3132 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1501762 T3132 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1501762 T3132 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
[junit4:junit4] 2> 1501763 T3132 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
[junit4:junit4] 2> 1501764 T3132 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
[junit4:junit4] 2> 1501764 T3132 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1501764 T3132 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty6-1369781926371/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty6/
[junit4:junit4] 2> 1501764 T3132 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@26dfe303
[junit4:junit4] 2> 1501765 T3132 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
[junit4:junit4] 2> 1501766 T3132 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty6
[junit4:junit4] 2> 1501767 T3132 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty6/index/
[junit4:junit4] 2> 1501768 T3132 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty6/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 1501769 T3132 oasc.CachingDirectoryFactory.get return new directory for /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty6/index
[junit4:junit4] 2> 1501775 T3132 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty6/index,segFN=segments_1,generation=1}
[junit4:junit4] 2> 1501775 T3132 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 1501778 T3132 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 1501778 T3132 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 1501779 T3132 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 1501780 T3132 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 1501780 T3132 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 1501781 T3132 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 1501781 T3132 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 1501781 T3132 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 1501782 T3132 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 1501794 T3132 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 1501805 T3132 oass.SolrIndexSearcher.<init> Opening Searcher@751d6ea6 main
[junit4:junit4] 2> 1501806 T3132 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 1501806 T3132 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 1501811 T3133 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@751d6ea6 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 1501813 T3132 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 1501813 T3132 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1502026 T3022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1502027 T3022 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"7",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:13456_sg_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:13456/sg_"}
[junit4:junit4] 2> 1502028 T3022 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
[junit4:junit4] 2> 1502028 T3022 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 1502039 T3028 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1502040 T3041 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1502039 T3112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1502040 T3074 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1502039 T3057 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1502039 T3021 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1502039 T3093 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1502039 T3131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 7)
[junit4:junit4] 2> 1502816 T3132 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 1502816 T3132 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:13456/sg_ collection:collection1 shard:shard1
[junit4:junit4] 2> 1502820 T3132 oasc.ZkController.register We are http://127.0.0.1:13456/sg_/collection1/ and leader is http://127.0.0.1:64652/sg_/collection1/
[junit4:junit4] 2> 1502820 T3132 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:13456/sg_
[junit4:junit4] 2> 1502820 T3132 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 1502820 T3132 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C930 name=collection1 org.apache.solr.core.SolrCore@e1d52a6 url=http://127.0.0.1:13456/sg_/collection1 node=127.0.0.1:13456_sg_ C930_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:13456_sg_, base_url=http://127.0.0.1:13456/sg_}
[junit4:junit4] 2> 1502821 T3134 C930 P13456 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 1502821 T3132 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 1502822 T3134 C930 P13456 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 1502822 T3134 C930 P13456 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 1502822 T3134 C930 P13456 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 1502823 T2999 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0
[junit4:junit4] 2> 1502823 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 1502823 T3134 C930 P13456 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1502824 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 1502842 T3035 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 7, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 1503087 T2999 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 1503090 T2999 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:24238
[junit4:junit4] 2> 1503091 T2999 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 1503091 T2999 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 1503092 T2999 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty7-1369781929714
[junit4:junit4] 2> 1503092 T2999 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty7-1369781929714/solr.xml
[junit4:junit4] 2> 1503093 T2999 oasc.CoreContainer.<init> New CoreContainer 1107392912
[junit4:junit4] 2> 1503093 T2999 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty7-1369781929714/'
[junit4:junit4] 2> 1503094 T2999 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/./org.apache.solr.cloud.SyncSliceTest-jetty7-1369781929714/'
[junit4:junit4] 2> 1503201 T2999 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 1503202 T2999 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 1503203 T2999 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 1503203 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 1503204 T2999 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 1503204 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 1503204 T2999 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 1503205 T2999 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 1503205 T2999 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 1503206 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 1503222 T2999 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 1503223 T2999 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63187/solr
[junit4:junit4] 2> 1503223 T2999 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 1503224 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1503227 T3146 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ffcb89e name:ZooKeeperConnection Watcher:127.0.0.1:63187 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1503227 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1503237 T2999 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 1503248 T2999 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 1503250 T3148 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2cbde4fa name:ZooKeeperConnection Watcher:127.0.0.1:63187/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 1503250 T2999 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 1503262 T2999 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> ASYNC NEW_CORE C931 name=collection1 org.apache.solr.core.SolrCore@7044b240 url=http://127.0.0.1:41961/sg_/collection1 node=127.0.0.1:41961_sg_ C931_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:41961_sg_, base_url=http://127.0.0.1:41961/sg_}
[junit4:junit4] 2> 1503521 T3117 C931 P41961 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:64652/sg_/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 1503521 T3117 C931 P41961 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:41961/sg_ START replicas=[http://127.0.0.1:64652/sg_/collection1/] nUpdates=100
[junit4:junit4] 2> 1503522 T3117 C931 P41961 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 1503522 T3117 C931 P41961 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 1503522 T3117 C931 P41961 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 1503523 T3117 C931 P41961 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 1503523 T3117 C931 P41961 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 1503523 T3117 C931 P41961 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:64652/sg_/collection1/. core=collection1
[junit4:junit4] 2> 1503523 T3117 C931 P41961 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C932 name=collection1 org.apache.solr.core.SolrCore@711e7e71 url=http://127.0.0.1:64652/sg_/collection1 node=127.0.0.1:64652_sg_ C932_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:64652_sg_, base_url=http://127.0.0.1:64652/sg_, leader=true}
[junit4:junit4] 2> 1503525 T3032 C932 P64652 oasc.SolrCore.execute [collection1] webapp=/sg_ path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 1503533 T3036 C932 P64652 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 1503538 T3036 C932 P64652 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=4
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_3,generation=3}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_4,generation=4}
[junit4:junit4] 2> commit{dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1369781902028/jetty1/index,segFN=segments_5,generation=5}
[junit4:junit4] 2> 1503538 T3036 C932 P64652 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
[junit4:junit4] 2> 1503540 T3036 C932 P64652 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 1503540 T3036 C932 P64652 oasup.LogUpdateProcessor.finish [collection1] webapp=/sg_ path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 7
[junit4:junit4] 2> 1503541 T3117 C931 P41961 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 1503541 T3117 C931 P41961 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 1503543 T3034 C932 P64652 oasc.
[...truncated too long message...]
[junit4:junit4] 1> "state":"active",
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "collection":"control_collection",
[junit4:junit4] 1> "node_name":"127.0.0.1:49557_sg_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:49557/sg_",
[junit4:junit4] 1> "leader":"true"}}}},
[junit4:junit4] 1> "router":"compositeId"}}
[junit4:junit4] 1> /solr/aliases.json (0)
[junit4:junit4] 1> /solr/live_nodes (7)
[junit4:junit4] 1> /solr/live_nodes/127.0.0.1:41961_sg_ (0)
[junit4:junit4] 1> /solr/live_nodes/127.0.0.1:51150_sg_ (0)
[junit4:junit4] 1> /solr/live_nodes/127.0.0.1:24238_sg_ (0)
[junit4:junit4] 1> /solr/live_nodes/127.0.0.1:13456_sg_ (0)
[junit4:junit4] 1> /solr/live_nodes/127.0.0.1:21322_sg_ (0)
[junit4:junit4] 1> /solr/live_nodes/127.0.0.1:64652_sg_ (0)
[junit4:junit4] 1> /solr/live_nodes/127.0.0.1:49557_sg_ (0)
[junit4:junit4] 1> /solr/overseer (3)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1>
[junit4:junit4] 1> /solr/overseer/queue (0)
[junit4:junit4] 1> /solr/overseer/queue-work (0)
[junit4:junit4] 1> /solr/overseer/collection-queue-work (0)
[junit4:junit4] 1> /solr/collections (2)
[junit4:junit4] 1> /solr/collections/collection1 (3)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {"configName":"conf1"}
[junit4:junit4] 1> /solr/collections/collection1/shards (0)
[junit4:junit4] 1> /solr/collections/collection1/leader_elect (1)
[junit4:junit4] 1> /solr/collections/collection1/leader_elect/shard1 (1)
[junit4:junit4] 1> /solr/collections/collection1/leader_elect/shard1/election (6)
[junit4:junit4] 1> /solr/collections/collection1/leader_elect/shard1/election/89770026731831306-4-n_0000000008 (0)
[junit4:junit4] 1> /solr/collections/collection1/leader_elect/shard1/election/89770026731831310-6-n_0000000004 (0)
[junit4:junit4] 1> /solr/collections/collection1/leader_elect/shard1/election/89770026731831312-7-n_0000000005 (0)
[junit4:junit4] 1> /solr/collections/collection1/leader_elect/shard1/election/89770026731831308-5-n_0000000003 (0)
[junit4:junit4] 1> /solr/collections/collection1/leader_elect/shard1/election/89770026731831314-8-n_0000000006 (0)
[junit4:junit4] 1> /solr/collections/collection1/leader_elect/shard1/election/89770026731831316-2-n_0000000007 (0)
[junit4:junit4] 1> /solr/collections/collection1/leaders (1)
[junit4:junit4] 1> /solr/collections/collection1/leaders/shard1 (0)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:51150_sg_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:51150/sg_"}
[junit4:junit4] 1> /solr/collections/control_collection (3)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {"configName":"conf1"}
[junit4:junit4] 1> /solr/collections/control_collection/shards (0)
[junit4:junit4] 1> /solr/collections/control_collection/leader_elect (1)
[junit4:junit4] 1> /solr/collections/control_collection/leader_elect/shard1 (1)
[junit4:junit4] 1> /solr/collections/control_collection/leader_elect/shard1/election (1)
[junit4:junit4] 1> /solr/collections/control_collection/leader_elect/shard1/election/89770026731831299-1-n_0000000000 (0)
[junit4:junit4] 1> /solr/collections/control_collection/leaders (1)
[junit4:junit4] 1> /solr/collections/control_collection/leaders/shard1 (0)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:49557_sg_",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:49557/sg_"}
[junit4:junit4] 1> /solr/overseer_elect (2)
[junit4:junit4] 1> /solr/overseer_elect/election (7)
[junit4:junit4] 1> /solr/overseer_elect/election/89770026731831310-127.0.0.1:41961_sg_-n_0000000005 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89770026731831316-127.0.0.1:64652_sg_-n_0000000008 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89770026731831312-127.0.0.1:13456_sg_-n_0000000006 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89770026731831314-127.0.0.1:24238_sg_-n_0000000007 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89770026731831299-127.0.0.1:49557_sg_-n_0000000000 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89770026731831306-127.0.0.1:21322_sg_-n_0000000003 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89770026731831308-127.0.0.1:51150_sg_-n_0000000004 (0)
[junit4:junit4] 1> /solr/overseer_elect/leader (0)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {"id":"89770026731831299-127.0.0.1:49557_sg_-n_0000000000"}
[junit4:junit4] 1> /zookeeper (1)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1>
[junit4:junit4] 1>
[junit4:junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=SyncSliceTest -Dtests.method=testDistribSearch -Dtests.seed=5629969592A39EB0 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt -Dtests.locale=ja -Dtests.timezone=Africa/Nairobi -Dtests.file.encoding=ISO8859-1
[junit4:junit4] FAILURE 902s J0 | SyncSliceTest.testDistribSearch <<<
[junit4:junit4] > Throwable #1: java.lang.AssertionError: shard1 is not consistent. Got 305 from http://127.0.0.1:41961/sg_/collection1lastClient and got 5 from http://127.0.0.1:13456/sg_/collection1
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([5629969592A39EB0:D7CF188DE5FCFE8C]:0)
[junit4:junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:963)
[junit4:junit4] > at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:238)
[junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:679)
[junit4:junit4] 2> 2376909 T2999 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4] 2> 901763 T2998 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
[junit4:junit4] 2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=ja, timezone=Africa/Nairobi
[junit4:junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Sun Microsystems Inc. 1.6.0_32 (64-bit)/cpus=16,threads=3,free=214778296,total=438501376
[junit4:junit4] 2> NOTE: All tests run in this JVM: [SolrCoreTest, UnloadDistributedZkTest, TestLMDirichletSimilarityFactory, TestClassNameShortening, RAMDirectoryFactoryTest, MoreLikeThisHandlerTest, SearchHandlerTest, TestFastLRUCache, UpdateRequestProcessorFactoryTest, BasicDistributedZkTest, TestDocSet, StatelessScriptUpdateProcessorFactoryTest, TestFieldCollectionResource, TestFieldTypeCollectionResource, AlternateDirectoryTest, TestSerializedLuceneMatchVersion, MBeansHandlerTest, TestCollationKeyRangeQueries, TestFieldResource, ShardRoutingTest, TestExtendedDismaxParser, TestFaceting, SuggesterTest, TestPHPSerializedResponseWriter, ResponseLogComponentTest, SystemInfoHandlerTest, PrimitiveFieldTypeTest, TermVectorComponentTest, WordBreakSolrSpellCheckerTest, DistributedTermsComponentTest, ZkCLITest, TestCloudManagedSchemaAddField, RegexBoostProcessorTest, CoreAdminHandlerTest, ShardSplitTest, ExternalFileFieldSortTest, TestIBSimilarityFactory, TestReversedWildcardFilterFactory, SynonymTokenizerTest, JsonLoaderTest, TestRealTimeGet, TestQueryUtils, ZkControllerTest, TestFunctionQuery, TestPhraseSuggestions, SpellingQueryConverterTest, TestQuerySenderListener, DocumentBuilderTest, ChaosMonkeyNothingIsSafeTest, RecoveryZkTest, TestValueSourceCache, ScriptEngineTest, AliasIntegrationTest, SpatialFilterTest, UUIDFieldTest, TestDocumentBuilder, PluginInfoTest, LoggingHandlerTest, TestSolrDeletionPolicy2, TestRandomDVFaceting, SOLR749Test, TestXIncludeConfig, ReturnFieldsTest, SolrInfoMBeanTest, MinimalSchemaTest, LeaderElectionTest, CacheHeaderTest, TestSchemaVersionResource, ResourceLoaderTest, QueryEqualityTest, ShowFileRequestHandlerTest, TestManagedSchema, BasicDistributedZk2Test, SyncSliceTest]
[junit4:junit4] Completed on J0 in 902.04s, 1 test, 1 failure <<< FAILURES!
[...truncated 448 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:393: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:366: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:181: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:437: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1243: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:887: There were test failures: 296 suites, 1235 tests, 1 failure, 11 ignored (5 assumptions)
Total time: 155 minutes 26 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 273 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/273/
1 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeyShardSplitTest.testDistribSearch
Error Message:
Server at http://127.0.0.1:54610 returned non ok status:500, message:Server Error
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:54610 returned non ok status:500, message:Server Error
at __randomizedtesting.SeedInfo.seed([D5E6A581AA59CC86:54002B99DD06ACBA]:0)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:208)
at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:136)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:679)
Build Log:
[...truncated 9679 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeyShardSplitTest
[junit4:junit4] 2> 2404762 T7747 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
[junit4:junit4] 2> 2404766 T7747 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solrtest-ChaosMonkeyShardSplitTest-1370037697155
[junit4:junit4] 2> 2404767 T7747 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 2404768 T7748 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 2404868 T7747 oasc.ZkTestServer.run start zk server on port:54600
[junit4:junit4] 2> 2404868 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2404873 T7754 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@378b361a name:ZooKeeperConnection Watcher:127.0.0.1:54600 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2404873 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2404874 T7747 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 2404887 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2404889 T7756 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2df28ad8 name:ZooKeeperConnection Watcher:127.0.0.1:54600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2404889 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2404890 T7747 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 2404899 T7747 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 2404901 T7747 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 2404903 T7747 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 2404911 T7747 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 2404911 T7747 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 2404917 T7747 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 2404917 T7747 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 2405021 T7747 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 2405022 T7747 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 2405032 T7747 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 2405033 T7747 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 2405036 T7747 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 2405037 T7747 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 2405040 T7747 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 2405041 T7747 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 2405043 T7747 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 2405044 T7747 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 2405047 T7747 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 2405048 T7747 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 2405055 T7747 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 2405055 T7747 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 2405315 T7747 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2405319 T7747 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54603
[junit4:junit4] 2> 2405320 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2405320 T7747 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2405321 T7747 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1370037697448
[junit4:junit4] 2> 2405321 T7747 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1370037697448/solr.xml
[junit4:junit4] 2> 2405322 T7747 oasc.CoreContainer.<init> New CoreContainer 41353038
[junit4:junit4] 2> 2405322 T7747 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1370037697448/'
[junit4:junit4] 2> 2405323 T7747 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1370037697448/'
[junit4:junit4] 2> 2405433 T7747 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2405433 T7747 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2405434 T7747 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2405434 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2405435 T7747 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2405435 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2405436 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2405436 T7747 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2405437 T7747 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2405437 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2405455 T7747 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2405456 T7747 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54600/solr
[junit4:junit4] 2> 2405456 T7747 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2405458 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2405460 T7767 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1365046d name:ZooKeeperConnection Watcher:127.0.0.1:54600 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2405460 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2405463 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2405476 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2405478 T7769 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@344d77b name:ZooKeeperConnection Watcher:127.0.0.1:54600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2405478 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2405480 T7747 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 2405489 T7747 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 2405493 T7747 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 2405500 T7747 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54603_
[junit4:junit4] 2> 2405502 T7747 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54603_
[junit4:junit4] 2> 2405505 T7747 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 2405519 T7747 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 2405527 T7747 oasc.Overseer.start Overseer (id=89786790521143299-127.0.0.1:54603_-n_0000000000) starting
[junit4:junit4] 2> 2405531 T7747 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 2405544 T7771 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 2405544 T7747 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 2405547 T7747 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 2405549 T7747 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2405551 T7770 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 2405555 T7772 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1370037697448/collection1
[junit4:junit4] 2> 2405555 T7772 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 2405556 T7772 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 2405556 T7772 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 2405557 T7772 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1370037697448/collection1/'
[junit4:junit4] 2> 2405559 T7772 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1370037697448/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 2405559 T7772 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1370037697448/collection1/lib/README' to classloader
[junit4:junit4] 2> 2405609 T7772 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 2405666 T7772 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 2405767 T7772 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 2405772 T7772 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2406324 T7772 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2406328 T7772 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2406330 T7772 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2406335 T7772 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2406354 T7772 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2406355 T7772 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1370037697448/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/control/data/
[junit4:junit4] 2> 2406355 T7772 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b784427
[junit4:junit4] 2> 2406356 T7772 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2406356 T7772 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/control/data
[junit4:junit4] 2> 2406356 T7772 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/control/data/index/
[junit4:junit4] 2> 2406357 T7772 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2406357 T7772 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/control/data/index
[junit4:junit4] 2> 2406361 T7772 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@3817a5ce lockFactory=org.apache.lucene.store.NativeFSLockFactory@75d8ce29; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> 2406362 T7772 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2406364 T7772 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2406365 T7772 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2406365 T7772 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2406366 T7772 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2406366 T7772 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2406366 T7772 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2406367 T7772 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2406367 T7772 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2406368 T7772 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2406377 T7772 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2406385 T7772 oass.SolrIndexSearcher.<init> Opening Searcher@70da8d04 main
[junit4:junit4] 2> 2406385 T7772 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/control/data/tlog
[junit4:junit4] 2> 2406386 T7772 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2406386 T7772 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2406390 T7773 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@70da8d04 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2406391 T7772 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2406392 T7772 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2407054 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2407055 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54603_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54603"}
[junit4:junit4] 2> 2407055 T7770 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 2407055 T7770 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 2407096 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 2407394 T7772 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2407394 T7772 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54603 collection:control_collection shard:shard1
[junit4:junit4] 2> 2407395 T7772 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 2407410 T7772 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 2407413 T7772 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 2407413 T7772 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 2407413 T7772 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54603/collection1/
[junit4:junit4] 2> 2407414 T7772 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 2407414 T7772 oasc.SyncStrategy.syncToMe http://127.0.0.1:54603/collection1/ has no replicas
[junit4:junit4] 2> 2407414 T7772 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54603/collection1/
[junit4:junit4] 2> 2407414 T7772 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 2408600 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2408623 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 2408644 T7772 oasc.ZkController.register We are http://127.0.0.1:54603/collection1/ and leader is http://127.0.0.1:54603/collection1/
[junit4:junit4] 2> 2408644 T7772 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54603
[junit4:junit4] 2> 2408644 T7772 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 2408644 T7772 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 2408645 T7772 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2408646 T7772 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2408647 T7747 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 2408648 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2408649 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2408666 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 2408668 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2408670 T7776 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@288be35e name:ZooKeeperConnection Watcher:127.0.0.1:54600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2408670 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2408672 T7747 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2408674 T7747 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 2408919 T7747 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2408922 T7747 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54610
[junit4:junit4] 2> 2408923 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2408923 T7747 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2408924 T7747 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1370037701063
[junit4:junit4] 2> 2408924 T7747 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1370037701063/solr.xml
[junit4:junit4] 2> 2408925 T7747 oasc.CoreContainer.<init> New CoreContainer 1050562978
[junit4:junit4] 2> 2408925 T7747 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1370037701063/'
[junit4:junit4] 2> 2408926 T7747 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1370037701063/'
[junit4:junit4] 2> 2409035 T7747 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2409036 T7747 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2409036 T7747 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2409036 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2409037 T7747 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2409037 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2409038 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2409038 T7747 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2409039 T7747 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2409039 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2409057 T7747 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2409057 T7747 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54600/solr
[junit4:junit4] 2> 2409058 T7747 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2409059 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2409063 T7787 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3732253c name:ZooKeeperConnection Watcher:127.0.0.1:54600 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2409064 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2409078 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2409090 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2409092 T7789 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@42b43e14 name:ZooKeeperConnection Watcher:127.0.0.1:54600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2409092 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2409103 T7747 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2410106 T7747 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54610_
[junit4:junit4] 2> 2410108 T7747 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54610_
[junit4:junit4] 2> 2410111 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 2410112 T7789 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 2410112 T7776 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 2410113 T7769 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 2410131 T7790 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1370037701063/collection1
[junit4:junit4] 2> 2410132 T7790 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 2410132 T7790 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 2410132 T7790 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 2410134 T7790 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1370037701063/collection1/'
[junit4:junit4] 2> 2410134 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2410135 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54603_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54603"}
[junit4:junit4] 2> 2410135 T7790 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1370037701063/collection1/lib/README' to classloader
[junit4:junit4] 2> 2410136 T7790 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1370037701063/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 2410144 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2410144 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2410144 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2410186 T7790 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 2410250 T7790 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 2410351 T7790 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 2410357 T7790 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2410910 T7790 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2410915 T7790 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2410917 T7790 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2410922 T7790 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2410942 T7790 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2410942 T7790 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1370037701063/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty1/
[junit4:junit4] 2> 2410943 T7790 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b784427
[junit4:junit4] 2> 2410943 T7790 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2410944 T7790 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty1
[junit4:junit4] 2> 2410944 T7790 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty1/index/
[junit4:junit4] 2> 2410944 T7790 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2410945 T7790 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty1/index
[junit4:junit4] 2> 2410949 T7790 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1fb81d8c lockFactory=org.apache.lucene.store.NativeFSLockFactory@215036bf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> 2410949 T7790 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2410952 T7790 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2410952 T7790 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2410952 T7790 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2410953 T7790 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2410954 T7790 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2410954 T7790 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2410954 T7790 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2410955 T7790 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2410955 T7790 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2410965 T7790 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2410972 T7790 oass.SolrIndexSearcher.<init> Opening Searcher@38527fdd main
[junit4:junit4] 2> 2410972 T7790 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty1/tlog
[junit4:junit4] 2> 2410974 T7790 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2410974 T7790 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2410978 T7791 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@38527fdd main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2410980 T7790 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2410980 T7790 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2411658 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2411659 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54610_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54610"}
[junit4:junit4] 2> 2411659 T7770 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4:junit4] 2> 2411659 T7770 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 2411672 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2411672 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2411672 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2411983 T7790 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2411983 T7790 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54610 collection:collection1 shard:shard1
[junit4:junit4] 2> 2411984 T7790 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 2411999 T7790 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 2412001 T7790 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 2412002 T7790 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 2412002 T7790 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54610/collection1/
[junit4:junit4] 2> 2412002 T7790 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 2412002 T7790 oasc.SyncStrategy.syncToMe http://127.0.0.1:54610/collection1/ has no replicas
[junit4:junit4] 2> 2412002 T7790 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54610/collection1/
[junit4:junit4] 2> 2412002 T7790 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 2413175 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2413196 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2413196 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2413196 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2413230 T7790 oasc.ZkController.register We are http://127.0.0.1:54610/collection1/ and leader is http://127.0.0.1:54610/collection1/
[junit4:junit4] 2> 2413230 T7790 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54610
[junit4:junit4] 2> 2413230 T7790 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 2413230 T7790 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 2413230 T7790 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2413255 T7790 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2413256 T7747 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 2413257 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2413257 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2413433 T7747 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2413436 T7747 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54613
[junit4:junit4] 2> 2413437 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2413437 T7747 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2413437 T7747 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1370037705656
[junit4:junit4] 2> 2413438 T7747 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1370037705656/solr.xml
[junit4:junit4] 2> 2413438 T7747 oasc.CoreContainer.<init> New CoreContainer 1361640916
[junit4:junit4] 2> 2413438 T7747 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1370037705656/'
[junit4:junit4] 2> 2413439 T7747 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1370037705656/'
[junit4:junit4] 2> 2413513 T7747 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2413514 T7747 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2413514 T7747 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2413515 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2413515 T7747 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2413516 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2413516 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2413517 T7747 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2413517 T7747 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2413518 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2413533 T7747 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2413534 T7747 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54600/solr
[junit4:junit4] 2> 2413535 T7747 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2413537 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2413566 T7803 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2afdf236 name:ZooKeeperConnection Watcher:127.0.0.1:54600 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2413567 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2413569 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2413577 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2413611 T7805 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b3fd0d9 name:ZooKeeperConnection Watcher:127.0.0.1:54600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2413611 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2413619 T7747 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2414622 T7747 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54613_
[junit4:junit4] 2> 2414649 T7747 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54613_
[junit4:junit4] 2> 2414652 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2414652 T7789 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 2414652 T7776 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 2414652 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2414652 T7805 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 2414653 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2414653 T7769 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 2414658 T7806 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1370037705656/collection1
[junit4:junit4] 2> 2414658 T7806 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 2414659 T7806 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 2414659 T7806 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 2414660 T7806 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1370037705656/collection1/'
[junit4:junit4] 2> 2414661 T7806 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1370037705656/collection1/lib/README' to classloader
[junit4:junit4] 2> 2414661 T7806 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1370037705656/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 2414695 T7806 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 2414713 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2414714 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54610_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54610"}
[junit4:junit4] 2> 2414726 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2414726 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2414726 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2414726 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2414733 T7806 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 2414834 T7806 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 2414838 T7806 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2415299 T7806 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2415303 T7806 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2415306 T7806 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2415311 T7806 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2415328 T7806 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2415328 T7806 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1370037705656/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty2/
[junit4:junit4] 2> 2415328 T7806 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b784427
[junit4:junit4] 2> 2415329 T7806 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2415330 T7806 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty2
[junit4:junit4] 2> 2415330 T7806 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty2/index/
[junit4:junit4] 2> 2415330 T7806 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2415331 T7806 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty2/index
[junit4:junit4] 2> 2415335 T7806 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7850654e lockFactory=org.apache.lucene.store.NativeFSLockFactory@6d45a4b1; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> 2415335 T7806 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2415338 T7806 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2415338 T7806 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2415339 T7806 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2415339 T7806 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2415340 T7806 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2415340 T7806 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2415340 T7806 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2415341 T7806 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2415341 T7806 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2415351 T7806 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2415358 T7806 oass.SolrIndexSearcher.<init> Opening Searcher@6d40d460 main
[junit4:junit4] 2> 2415358 T7806 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty2/tlog
[junit4:junit4] 2> 2415359 T7806 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2415359 T7806 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2415364 T7807 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6d40d460 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2415366 T7806 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2415366 T7806 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2416230 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2416231 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54613_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54613"}
[junit4:junit4] 2> 2416231 T7770 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 2416231 T7770 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 2416261 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2416261 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2416261 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2416261 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2416394 T7806 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2416394 T7806 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54613 collection:collection1 shard:shard2
[junit4:junit4] 2> 2416395 T7806 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 2416414 T7806 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 2416424 T7806 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 2416424 T7806 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 2416424 T7806 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54613/collection1/
[junit4:junit4] 2> 2416425 T7806 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 2416425 T7806 oasc.SyncStrategy.syncToMe http://127.0.0.1:54613/collection1/ has no replicas
[junit4:junit4] 2> 2416425 T7806 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54613/collection1/
[junit4:junit4] 2> 2416425 T7806 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 2417773 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2417791 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2417791 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2417791 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2417791 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2417815 T7806 oasc.ZkController.register We are http://127.0.0.1:54613/collection1/ and leader is http://127.0.0.1:54613/collection1/
[junit4:junit4] 2> 2417815 T7806 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54613
[junit4:junit4] 2> 2417815 T7806 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 2417815 T7806 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 2417815 T7806 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2417817 T7806 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2417818 T7747 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 2417818 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2417818 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2417994 T7747 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2417996 T7747 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54616
[junit4:junit4] 2> 2417997 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2417997 T7747 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2417997 T7747 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1370037710216
[junit4:junit4] 2> 2417998 T7747 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1370037710216/solr.xml
[junit4:junit4] 2> 2417998 T7747 oasc.CoreContainer.<init> New CoreContainer 844009891
[junit4:junit4] 2> 2417999 T7747 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1370037710216/'
[junit4:junit4] 2> 2417999 T7747 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1370037710216/'
[junit4:junit4] 2> 2418071 T7747 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2418072 T7747 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2418072 T7747 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2418072 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2418073 T7747 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2418073 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2418073 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2418073 T7747 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2418074 T7747 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2418074 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2418084 T7747 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2418085 T7747 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54600/solr
[junit4:junit4] 2> 2418085 T7747 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2418086 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2418088 T7819 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1dd60d2d name:ZooKeeperConnection Watcher:127.0.0.1:54600 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2418089 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2418104 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2418113 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2418114 T7821 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6220535d name:ZooKeeperConnection Watcher:127.0.0.1:54600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2418115 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2418126 T7747 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2419129 T7747 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54616_
[junit4:junit4] 2> 2419131 T7747 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54616_
[junit4:junit4] 2> 2419134 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2419134 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 2419135 T7821 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2419135 T7776 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2419135 T7789 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2419135 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2419135 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2419136 T7805 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2419146 T7769 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 2419151 T7822 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1370037710216/collection1
[junit4:junit4] 2> 2419151 T7822 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 2419152 T7822 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 2419152 T7822 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 2419153 T7822 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1370037710216/collection1/'
[junit4:junit4] 2> 2419155 T7822 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1370037710216/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 2419155 T7822 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1370037710216/collection1/lib/README' to classloader
[junit4:junit4] 2> 2419206 T7822 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 2419263 T7822 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 2419295 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2419296 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54613_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54613"}
[junit4:junit4] 2> 2419308 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2419309 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2419309 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2419309 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2419364 T7822 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 2419364 T7821 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2419370 T7822 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2419928 T7822 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2419932 T7822 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2419935 T7822 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2419940 T7822 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2419958 T7822 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2419958 T7822 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1370037710216/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/
[junit4:junit4] 2> 2419959 T7822 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b784427
[junit4:junit4] 2> 2419959 T7822 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2419961 T7822 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3
[junit4:junit4] 2> 2419961 T7822 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/index/
[junit4:junit4] 2> 2419961 T7822 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2419962 T7822 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/index
[junit4:junit4] 2> 2419966 T7822 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7f59ee8a lockFactory=org.apache.lucene.store.NativeFSLockFactory@97305cf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> 2419966 T7822 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2419969 T7822 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2419969 T7822 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2419970 T7822 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2419970 T7822 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2419971 T7822 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2419971 T7822 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2419972 T7822 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2419972 T7822 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2419973 T7822 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2419982 T7822 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2419989 T7822 oass.SolrIndexSearcher.<init> Opening Searcher@a280e2a main
[junit4:junit4] 2> 2419990 T7822 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/tlog
[junit4:junit4] 2> 2419991 T7822 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2419991 T7822 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2419996 T7823 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a280e2a main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2419997 T7822 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2419998 T7822 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2420813 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2420814 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54616_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54616"}
[junit4:junit4] 2> 2420814 T7770 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 2420814 T7770 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 2420836 T7821 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2420836 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2420836 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2420836 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2420836 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2421029 T7822 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2421029 T7822 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54616 collection:collection1 shard:shard1
[junit4:junit4] 2> 2421032 T7822 oasc.ZkController.register We are http://127.0.0.1:54616/collection1/ and leader is http://127.0.0.1:54610/collection1/
[junit4:junit4] 2> 2421032 T7822 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54616
[junit4:junit4] 2> 2421032 T7822 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 2421032 T7822 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C22595 name=collection1 org.apache.solr.core.SolrCore@4cf55f36 url=http://127.0.0.1:54616/collection1 node=127.0.0.1:54616_ C22595_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:54616_, base_url=http://127.0.0.1:54616}
[junit4:junit4] 2> 2421033 T7824 C22595 P54616 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 2421033 T7822 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2421034 T7824 C22595 P54616 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 2421034 T7824 C22595 P54616 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 2421035 T7824 C22595 P54616 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2421035 T7747 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 2421035 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2421036 T7824 C22595 P54616 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2421036 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2421055 T7779 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 2421221 T7747 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 2421223 T7747 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54620
[junit4:junit4] 2> 2421223 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 2421224 T7747 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 2421224 T7747 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1370037713442
[junit4:junit4] 2> 2421225 T7747 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1370037713442/solr.xml
[junit4:junit4] 2> 2421225 T7747 oasc.CoreContainer.<init> New CoreContainer 613007754
[junit4:junit4] 2> 2421225 T7747 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1370037713442/'
[junit4:junit4] 2> 2421226 T7747 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1370037713442/'
[junit4:junit4] 2> 2421299 T7747 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 2421300 T7747 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 2421300 T7747 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 2421300 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 2421301 T7747 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 2421301 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 2421301 T7747 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 2421302 T7747 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 2421302 T7747 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 2421302 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 2421314 T7747 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 2421315 T7747 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54600/solr
[junit4:junit4] 2> 2421315 T7747 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 2421316 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2421319 T7836 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9b2d63e name:ZooKeeperConnection Watcher:127.0.0.1:54600 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2421319 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2421332 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 2421341 T7747 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 2421343 T7838 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6972acaa name:ZooKeeperConnection Watcher:127.0.0.1:54600/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 2421343 T7747 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 2421354 T7747 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 2422340 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2422341 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54616_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54616"}
[junit4:junit4] 2> 2422355 T7821 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2422355 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2422355 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2422355 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2422355 T7838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2422355 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2422358 T7747 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54620_
[junit4:junit4] 2> 2422363 T7747 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54620_
[junit4:junit4] 2> 2422365 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2422365 T7838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2422365 T7821 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2422365 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 2422366 T7821 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2422365 T7776 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2422365 T7789 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2422372 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2422372 T7838 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2422372 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2422372 T7805 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2422373 T7769 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 2422377 T7839 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1370037713442/collection1
[junit4:junit4] 2> 2422377 T7839 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 2422378 T7839 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 2422378 T7839 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 2422379 T7839 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1370037713442/collection1/'
[junit4:junit4] 2> 2422380 T7839 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1370037713442/collection1/lib/README' to classloader
[junit4:junit4] 2> 2422380 T7839 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1370037713442/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 2422414 T7839 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 2422452 T7839 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 2422553 T7839 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 2422557 T7839 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 2422934 T7839 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 2422937 T7839 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 2422939 T7839 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2422943 T7839 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 2422957 T7839 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2422957 T7839 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1370037713442/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/
[junit4:junit4] 2> 2422957 T7839 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b784427
[junit4:junit4] 2> 2422957 T7839 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 2422958 T7839 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4
[junit4:junit4] 2> 2422958 T7839 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index/
[junit4:junit4] 2> 2422958 T7839 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 2422959 T7839 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index
[junit4:junit4] 2> 2422962 T7839 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7d75fa26 lockFactory=org.apache.lucene.store.NativeFSLockFactory@47d79938; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> 2422962 T7839 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2422964 T7839 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 2422964 T7839 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 2422964 T7839 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 2422965 T7839 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 2422965 T7839 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 2422966 T7839 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 2422966 T7839 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 2422966 T7839 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 2422967 T7839 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 2422973 T7839 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 2422978 T7839 oass.SolrIndexSearcher.<init> Opening Searcher@2959709b main
[junit4:junit4] 2> 2422978 T7839 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/tlog
[junit4:junit4] 2> 2422979 T7839 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 2422979 T7839 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 2422982 T7840 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2959709b main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 2422983 T7839 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 2422984 T7839 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2423058 T7779 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true for: 2 seconds.
[junit4:junit4] 2> 2423058 T7779 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=4&state=recovering&nodeName=127.0.0.1:54616_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=2003
[junit4:junit4] 2> 2423860 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2423861 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54620_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54620"}
[junit4:junit4] 2> 2423861 T7770 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 2423861 T7770 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 2423882 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2423883 T7821 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2423882 T7838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2423882 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2423882 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2423882 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2423986 T7839 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 2423986 T7839 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54620 collection:collection1 shard:shard2
[junit4:junit4] 2> 2423989 T7839 oasc.ZkController.register We are http://127.0.0.1:54620/collection1/ and leader is http://127.0.0.1:54613/collection1/
[junit4:junit4] 2> 2423989 T7839 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54620
[junit4:junit4] 2> 2423989 T7839 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 2423990 T7839 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C22596 name=collection1 org.apache.solr.core.SolrCore@6e6625d2 url=http://127.0.0.1:54620/collection1 node=127.0.0.1:54620_ C22596_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:54620_, base_url=http://127.0.0.1:54620}
[junit4:junit4] 2> 2423990 T7841 C22596 P54620 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 2423991 T7841 C22596 P54620 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 2423991 T7839 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2423991 T7841 C22596 P54620 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 2423991 T7841 C22596 P54620 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2423992 T7747 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 2423992 T7747 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 2423992 T7841 C22596 P54620 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2423993 T7747 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2424005 T7747 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2424006 T7795 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 2424006 T7747 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 15 for each attempt
[junit4:junit4] 2> 2424006 T7747 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4:junit4] 2> 2424007 T7747 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2425008 T7747 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> ASYNC NEW_CORE C22597 name=collection1 org.apache.solr.core.SolrCore@4cf55f36 url=http://127.0.0.1:54616/collection1 node=127.0.0.1:54616_ C22597_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:54616_, base_url=http://127.0.0.1:54616}
[junit4:junit4] 2> 2425060 T7824 C22597 P54616 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:54610/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 2425060 T7824 C22597 P54616 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:54616 START replicas=[http://127.0.0.1:54610/collection1/] nUpdates=100
[junit4:junit4] 2> 2425061 T7824 C22597 P54616 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 2425061 T7824 C22597 P54616 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 2425062 T7824 C22597 P54616 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 2425062 T7824 C22597 P54616 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 2425062 T7824 C22597 P54616 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 2425062 T7824 C22597 P54616 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:54610/collection1/. core=collection1
[junit4:junit4] 2> 2425063 T7824 C22597 P54616 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C22598 name=collection1 org.apache.solr.core.SolrCore@5eee1d3a url=http://127.0.0.1:54610/collection1 node=127.0.0.1:54610_ C22598_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:54610_, base_url=http://127.0.0.1:54610, leader=true}
[junit4:junit4] 2> 2425069 T7780 C22598 P54610 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 2425074 T7781 C22598 P54610 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 2425075 T7781 C22598 P54610 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1fb81d8c lockFactory=org.apache.lucene.store.NativeFSLockFactory@215036bf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> 2425076 T7781 C22598 P54610 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2425076 T7781 C22598 P54610 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1fb81d8c lockFactory=org.apache.lucene.store.NativeFSLockFactory@215036bf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1fb81d8c lockFactory=org.apache.lucene.store.NativeFSLockFactory@215036bf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_2,generation=2}
[junit4:junit4] 2> 2425077 T7781 C22598 P54610 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2425077 T7781 C22598 P54610 oass.SolrIndexSearcher.<init> Opening Searcher@89bfe1e realtime
[junit4:junit4] 2> 2425077 T7781 C22598 P54610 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2425078 T7781 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 4
[junit4:junit4] 2> 2425078 T7824 C22597 P54616 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2425079 T7824 C22597 P54616 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 2425080 T7782 C22598 P54610 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2425081 T7782 C22598 P54610 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2
[junit4:junit4] 2> 2425081 T7824 C22597 P54616 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 2425081 T7824 C22597 P54616 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 2425082 T7824 C22597 P54616 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 2425084 T7782 C22598 P54610 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 2425085 T7824 C22597 P54616 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 2425086 T7824 C22597 P54616 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/index.20130601000157473
[junit4:junit4] 2> 2425086 T7824 C22597 P54616 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@4edaa90d lockFactory=org.apache.lucene.store.NativeFSLockFactory@34ce2de8) fullCopy=false
[junit4:junit4] 2> 2425088 T7782 C22598 P54610 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 2425089 T7824 C22597 P54616 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 2425089 T7824 C22597 P54616 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 2425090 T7824 C22597 P54616 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 2425092 T7824 C22597 P54616 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7f59ee8a lockFactory=org.apache.lucene.store.NativeFSLockFactory@97305cf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7f59ee8a lockFactory=org.apache.lucene.store.NativeFSLockFactory@97305cf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_2,generation=2}
[junit4:junit4] 2> 2425092 T7824 C22597 P54616 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2425092 T7824 C22597 P54616 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 2425093 T7824 C22597 P54616 oass.SolrIndexSearcher.<init> Opening Searcher@488fb805 main
[junit4:junit4] 2> 2425094 T7823 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@488fb805 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2425094 T7824 C22597 P54616 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/index.20130601000157473 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/index.20130601000157473;done=true>>]
[junit4:junit4] 2> 2425094 T7824 C22597 P54616 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/index.20130601000157473
[junit4:junit4] 2> 2425095 T7824 C22597 P54616 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty3/index.20130601000157473
[junit4:junit4] 2> 2425095 T7824 C22597 P54616 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 2425095 T7824 C22597 P54616 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 2425095 T7824 C22597 P54616 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 2425095 T7824 C22597 P54616 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2425097 T7824 C22597 P54616 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 2425386 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2425387 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54620_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54620"}
[junit4:junit4] 2> 2425390 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54616_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54616"}
[junit4:junit4] 2> 2425405 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2425405 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2425405 T7821 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2425405 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2425405 T7838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2425405 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2426008 T7795 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 2 seconds.
[junit4:junit4] 2> 2426008 T7795 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:54620_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=2002
[junit4:junit4] 2> 2426010 T7747 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2427012 T7747 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> C22596_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:54620_, base_url=http://127.0.0.1:54620}
[junit4:junit4] 2> 2428010 T7841 C22596 P54620 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:54613/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 2428010 T7841 C22596 P54620 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:54620 START replicas=[http://127.0.0.1:54613/collection1/] nUpdates=100
[junit4:junit4] 2> 2428010 T7841 C22596 P54620 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 2428011 T7841 C22596 P54620 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 2428011 T7841 C22596 P54620 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 2428011 T7841 C22596 P54620 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 2428011 T7841 C22596 P54620 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 2428011 T7841 C22596 P54620 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:54613/collection1/. core=collection1
[junit4:junit4] 2> 2428011 T7841 C22596 P54620 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 2428014 T7747 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> ASYNC NEW_CORE C22599 name=collection1 org.apache.solr.core.SolrCore@22446f12 url=http://127.0.0.1:54613/collection1 node=127.0.0.1:54613_ C22599_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:54613_, base_url=http://127.0.0.1:54613, leader=true}
[junit4:junit4] 2> 2428017 T7796 C22599 P54613 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1
[junit4:junit4] 2> 2428020 T7797 C22599 P54613 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 2428021 T7797 C22599 P54613 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7850654e lockFactory=org.apache.lucene.store.NativeFSLockFactory@6d45a4b1; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> 2428022 T7797 C22599 P54613 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2428022 T7797 C22599 P54613 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7850654e lockFactory=org.apache.lucene.store.NativeFSLockFactory@6d45a4b1; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7850654e lockFactory=org.apache.lucene.store.NativeFSLockFactory@6d45a4b1; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_2,generation=2}
[junit4:junit4] 2> 2428023 T7797 C22599 P54613 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2428023 T7797 C22599 P54613 oass.SolrIndexSearcher.<init> Opening Searcher@51d5ac38 realtime
[junit4:junit4] 2> 2428023 T7797 C22599 P54613 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2428023 T7797 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 3
[junit4:junit4] 2> 2428024 T7841 C22596 P54620 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2428024 T7841 C22596 P54620 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 2428025 T7798 C22599 P54613 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 2428025 T7798 C22599 P54613 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=0
[junit4:junit4] 2> 2428026 T7841 C22596 P54620 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 2428026 T7841 C22596 P54620 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 2428026 T7841 C22596 P54620 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 2428028 T7798 C22599 P54613 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 2428028 T7841 C22596 P54620 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 2428029 T7841 C22596 P54620 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index.20130601000200416
[junit4:junit4] 2> 2428029 T7841 C22596 P54620 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@4eeec90c lockFactory=org.apache.lucene.store.NativeFSLockFactory@4ab4d5ef) fullCopy=false
[junit4:junit4] 2> 2428031 T7798 C22599 P54613 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 2428032 T7841 C22596 P54620 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 2428032 T7841 C22596 P54620 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 2428032 T7841 C22596 P54620 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 2428033 T7841 C22596 P54620 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7d75fa26 lockFactory=org.apache.lucene.store.NativeFSLockFactory@47d79938; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7d75fa26 lockFactory=org.apache.lucene.store.NativeFSLockFactory@47d79938; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_2,generation=2}
[junit4:junit4] 2> 2428034 T7841 C22596 P54620 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2428034 T7841 C22596 P54620 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 2428034 T7841 C22596 P54620 oass.SolrIndexSearcher.<init> Opening Searcher@5673d73a main
[junit4:junit4] 2> 2428035 T7840 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5673d73a main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2428035 T7841 C22596 P54620 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index.20130601000200416 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index.20130601000200416;done=true>>]
[junit4:junit4] 2> 2428035 T7841 C22596 P54620 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index.20130601000200416
[junit4:junit4] 2> 2428036 T7841 C22596 P54620 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index.20130601000200416
[junit4:junit4] 2> 2428036 T7841 C22596 P54620 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 2428036 T7841 C22596 P54620 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 2428036 T7841 C22596 P54620 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 2428036 T7841 C22596 P54620 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 2428038 T7841 C22596 P54620 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 2428412 T7770 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2428413 T7770 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54620_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54620"}
[junit4:junit4] 2> 2428481 T7776 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2428481 T7821 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2428481 T7789 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2428481 T7805 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2428481 T7838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2428481 T7769 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 2429016 T7747 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2429017 T7747 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
[junit4:junit4] 2> ASYNC NEW_CORE C22600 name=collection1 org.apache.solr.core.SolrCore@4b03da26 url=http://127.0.0.1:54603/collection1 node=127.0.0.1:54603_ C22600_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:54603_, base_url=http://127.0.0.1:54603, leader=true}
[junit4:junit4] 2> 2429023 T7759 C22600 P54603 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 2429025 T7759 C22600 P54603 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@3817a5ce lockFactory=org.apache.lucene.store.NativeFSLockFactory@75d8ce29; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> 2429026 T7759 C22600 P54603 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 2429026 T7759 C22600 P54603 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@3817a5ce lockFactory=org.apache.lucene.store.NativeFSLockFactory@75d8ce29; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@3817a5ce lockFactory=org.apache.lucene.store.NativeFSLockFactory@75d8ce29; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_2,generation=2}
[junit4:junit4] 2> 2429027 T7759 C22600 P54603 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 2429027 T7759 C22600 P54603 oass.SolrIndexSearcher.<init> Opening Searcher@4752f407 main
[junit4:junit4] 2> 2429027 T7759 C22600 P54603 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2429028 T7773 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4752f407 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2429029 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 6
[junit4:junit4] 2> 2429040 T7783 C22598 P54610 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 2429042 T7783 C22598 P54610 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1fb81d8c lockFactory=org.apache.lucene.store.NativeFSLockFactory@215036bf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1fb81d8c lockFactory=org.apache.lucene.store.NativeFSLockFactory@215036bf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_3,generation=3}
[junit4:junit4] 2> 2429042 T7783 C22598 P54610 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 2429043 T7783 C22598 P54610 oass.SolrIndexSearcher.<init> Opening Searcher@686492a5 main
[junit4:junit4] 2> 2429043 T7783 C22598 P54610 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2429044 T7791 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@686492a5 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2429044 T7783 C22598 P54610 oasu.SolrCmdDistributor.distribCommit Distrib commit to:[StdNode: http://127.0.0.1:54616/collection1/, StdNode: http://127.0.0.1:54613/collection1/, StdNode: http://127.0.0.1:54620/collection1/] params:commit_end_point=true&commit=true&softCommit=false&waitSearcher=true&expungeDeletes=false
[junit4:junit4] 2> 2429047 T7799 C22599 P54613 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 2429048 T7799 C22599 P54613 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7850654e lockFactory=org.apache.lucene.store.NativeFSLockFactory@6d45a4b1; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7850654e lockFactory=org.apache.lucene.store.NativeFSLockFactory@6d45a4b1; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_3,generation=3}
[junit4:junit4] 2> 2429048 T7799 C22599 P54613 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 2429049 T7799 C22599 P54613 oass.SolrIndexSearcher.<init> Opening Searcher@7f3410db main
[junit4:junit4] 2> 2429049 T7799 C22599 P54613 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2429050 T7807 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f3410db main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2429050 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 3
[junit4:junit4] 2> C22596_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:54620_, base_url=http://127.0.0.1:54620}
[junit4:junit4] 2> 2429052 T7828 C22596 P54620 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> C22597_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:54616_, base_url=http://127.0.0.1:54616}
[junit4:junit4] 2> 2429052 T7811 C22597 P54616 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 2429053 T7828 C22596 P54620 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7d75fa26 lockFactory=org.apache.lucene.store.NativeFSLockFactory@47d79938; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7d75fa26 lockFactory=org.apache.lucene.store.NativeFSLockFactory@47d79938; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_3,generation=3}
[junit4:junit4] 2> 2429053 T7811 C22597 P54616 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7f59ee8a lockFactory=org.apache.lucene.store.NativeFSLockFactory@97305cf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@7f59ee8a lockFactory=org.apache.lucene.store.NativeFSLockFactory@97305cf; maxCacheMB=0.62890625 maxMergeSizeMB=0.3837890625)),segFN=segments_3,generation=3}
[junit4:junit4] 2> 2429053 T7828 C22596 P54620 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 2429053 T7811 C22597 P54616 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 2429054 T7828 C22596 P54620 oass.SolrIndexSearcher.<init> Opening Searcher@42024574 main
[junit4:junit4] 2> 2429054 T7811 C22597 P54616 oass.SolrIndexSearcher.<init> Opening Searcher@7f866d13 main
[junit4:junit4] 2> 2429054 T7828 C22596 P54620 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2429055 T7811 C22597 P54616 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 2429055 T7840 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@42024574 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2429055 T7823 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f866d13 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 2429056 T7828 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 4
[junit4:junit4] 2> 2429056 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 4
[junit4:junit4] 2> 2429056 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 16
[junit4:junit4] 2> 2429057 T7747 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 2429059 T7784 C22598 P54610 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 2429061 T7812 C22597 P54616 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 2429062 T7800 C22599 P54613 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 2429063 T7829 C22596 P54620 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 2431068 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1436588675909550080)} 0 1
[junit4:junit4] 2> 2431074 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1436588675912695808&update.from=http://127.0.0.1:54613/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1436588675912695808)} 0 1
[junit4:junit4] 2> 2431077 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1436588675915841536&update.from=http://127.0.0.1:54610/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1436588675915841536)} 0 1
[junit4:junit4] 2> 2431078 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=TOLEADER&wt=javabin&version=2} {deleteByQuery=*:* (-1436588675915841536)} 0 5
[junit4:junit4] 2> 2431078 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1436588675912695808)} 0 8
[junit4:junit4] 2> 2431081 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[0 (1436588675924230144)]} 0 0
[junit4:junit4] 2> 2431088 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[0 (1436588675929473024)]} 0 0
[junit4:junit4] 2> 2431089 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[0 (1436588675929473024)]} 0 3
[junit4:junit4] 2> 2431089 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[0]} 0 5
[junit4:junit4] 2> 2431091 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[1 (1436588675934715904)]} 0 0
[junit4:junit4] 2> 2431097 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[1 (1436588675938910208)]} 0 0
[junit4:junit4] 2> 2431097 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[1 (1436588675938910208)]} 0 2
[junit4:junit4] 2> 2431098 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[1]} 0 5
[junit4:junit4] 2> 2431100 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[2 (1436588675944153088)]} 0 0
[junit4:junit4] 2> 2431104 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[2 (1436588675946250240)]} 0 0
[junit4:junit4] 2> 2431105 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[2 (1436588675946250240)]} 0 3
[junit4:junit4] 2> 2431107 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[3 (1436588675951493120)]} 0 0
[junit4:junit4] 2> 2431113 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[3 (1436588675955687424)]} 0 0
[junit4:junit4] 2> 2431113 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[3 (1436588675955687424)]} 0 2
[junit4:junit4] 2> 2431114 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[3]} 0 5
[junit4:junit4] 2> 2431116 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[4 (1436588675960930304)]} 0 0
[junit4:junit4] 2> 2431122 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[4 (1436588675965124608)]} 0 0
[junit4:junit4] 2> 2431122 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[4 (1436588675965124608)]} 0 2
[junit4:junit4] 2> 2431123 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[4]} 0 5
[junit4:junit4] 2> 2431125 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[5 (1436588675970367488)]} 0 0
[junit4:junit4] 2> 2431129 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[5 (1436588675972464640)]} 0 0
[junit4:junit4] 2> 2431129 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[5 (1436588675972464640)]} 0 2
[junit4:junit4] 2> 2431132 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[6 (1436588675977707520)]} 0 0
[junit4:junit4] 2> 2431136 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[6 (1436588675979804672)]} 0 0
[junit4:junit4] 2> 2431136 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[6 (1436588675979804672)]} 0 2
[junit4:junit4] 2> 2431139 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[7 (1436588675985047552)]} 0 0
[junit4:junit4] 2> 2431145 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[7 (1436588675989241856)]} 0 0
[junit4:junit4] 2> 2431145 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[7 (1436588675989241856)]} 0 2
[junit4:junit4] 2> 2431146 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[7]} 0 5
[junit4:junit4] 2> 2431148 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[8 (1436588675994484736)]} 0 0
[junit4:junit4] 2> 2431154 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[8 (1436588675998679040)]} 0 0
[junit4:junit4] 2> 2431154 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[8 (1436588675998679040)]} 0 2
[junit4:junit4] 2> 2431155 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[8]} 0 5
[junit4:junit4] 2> 2431157 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[9 (1436588676003921920)]} 0 0
[junit4:junit4] 2> 2431162 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[9 (1436588676007067648)]} 0 0
[junit4:junit4] 2> 2431162 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[9 (1436588676007067648)]} 0 2
[junit4:junit4] 2> 2431165 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[10 (1436588676012310528)]} 0 0
[junit4:junit4] 2> 2431169 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[10 (1436588676014407680)]} 0 0
[junit4:junit4] 2> 2431170 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[10 (1436588676014407680)]} 0 3
[junit4:junit4] 2> 2431172 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[11 (1436588676019650560)]} 0 0
[junit4:junit4] 2> 2431177 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[11 (1436588676022796288)]} 0 0
[junit4:junit4] 2> 2431178 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[11 (1436588676022796288)]} 0 3
[junit4:junit4] 2> 2431180 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[12 (1436588676028039168)]} 0 0
[junit4:junit4] 2> 2431187 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[12 (1436588676033282048)]} 0 0
[junit4:junit4] 2> 2431187 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[12 (1436588676033282048)]} 0 2
[junit4:junit4] 2> 2431188 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[12]} 0 5
[junit4:junit4] 2> 2431190 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[13 (1436588676038524928)]} 0 0
[junit4:junit4] 2> 2431194 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[13 (1436588676040622080)]} 0 0
[junit4:junit4] 2> 2431194 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[13 (1436588676040622080)]} 0 2
[junit4:junit4] 2> 2431197 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[14 (1436588676045864960)]} 0 0
[junit4:junit4] 2> 2431201 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[14 (1436588676047962112)]} 0 0
[junit4:junit4] 2> 2431201 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[14 (1436588676047962112)]} 0 2
[junit4:junit4] 2> 2431204 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[15 (1436588676053204992)]} 0 0
[junit4:junit4] 2> 2431210 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[15 (1436588676057399296)]} 0 0
[junit4:junit4] 2> 2431210 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[15 (1436588676057399296)]} 0 2
[junit4:junit4] 2> 2431211 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[15]} 0 5
[junit4:junit4] 2> 2431213 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[16 (1436588676062642176)]} 0 0
[junit4:junit4] 2> 2431219 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[16 (1436588676066836480)]} 0 0
[junit4:junit4] 2> 2431220 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[16 (1436588676066836480)]} 0 3
[junit4:junit4] 2> 2431220 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[16]} 0 5
[junit4:junit4] 2> 2431223 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[17 (1436588676073127936)]} 0 0
[junit4:junit4] 2> 2431230 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[17 (1436588676078370816)]} 0 0
[junit4:junit4] 2> 2431231 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[17 (1436588676078370816)]} 0 3
[junit4:junit4] 2> 2431231 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[17]} 0 5
[junit4:junit4] 2> 2431233 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[18 (1436588676083613696)]} 0 0
[junit4:junit4] 2> 2431239 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[18 (1436588676087808000)]} 0 0
[junit4:junit4] 2> 2431239 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[18 (1436588676087808000)]} 0 2
[junit4:junit4] 2> 2431240 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[18]} 0 5
[junit4:junit4] 2> 2431242 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[19 (1436588676093050880)]} 0 0
[junit4:junit4] 2> 2431246 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[19 (1436588676095148032)]} 0 0
[junit4:junit4] 2> 2431246 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[19 (1436588676095148032)]} 0 2
[junit4:junit4] 2> 2431249 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[20 (1436588676100390912)]} 0 0
[junit4:junit4] 2> 2431253 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[20 (1436588676102488064)]} 0 0
[junit4:junit4] 2> 2431253 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[20 (1436588676102488064)]} 0 2
[junit4:junit4] 2> 2431256 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[21 (1436588676107730944)]} 0 0
[junit4:junit4] 2> 2431262 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[21 (1436588676111925248)]} 0 0
[junit4:junit4] 2> 2431262 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[21 (1436588676111925248)]} 0 2
[junit4:junit4] 2> 2431263 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[21]} 0 5
[junit4:junit4] 2> 2431265 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[22 (1436588676117168128)]} 0 0
[junit4:junit4] 2> 2431271 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[22 (1436588676121362432)]} 0 0
[junit4:junit4] 2> 2431271 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[22 (1436588676121362432)]} 0 2
[junit4:junit4] 2> 2431272 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[22]} 0 5
[junit4:junit4] 2> 2431274 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[23 (1436588676126605312)]} 0 0
[junit4:junit4] 2> 2431278 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[23 (1436588676128702464)]} 0 0
[junit4:junit4] 2> 2431278 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[23 (1436588676128702464)]} 0 2
[junit4:junit4] 2> 2431281 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[24 (1436588676133945344)]} 0 0
[junit4:junit4] 2> 2431285 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[24 (1436588676136042496)]} 0 0
[junit4:junit4] 2> 2431286 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[24 (1436588676136042496)]} 0 3
[junit4:junit4] 2> 2431288 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[25 (1436588676141285376)]} 0 0
[junit4:junit4] 2> 2431294 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[25 (1436588676145479680)]} 0 0
[junit4:junit4] 2> 2431294 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[25 (1436588676145479680)]} 0 2
[junit4:junit4] 2> 2431295 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[25]} 0 5
[junit4:junit4] 2> 2431297 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[26 (1436588676150722560)]} 0 0
[junit4:junit4] 2> 2431304 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[26 (1436588676155965440)]} 0 0
[junit4:junit4] 2> 2431304 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[26 (1436588676155965440)]} 0 2
[junit4:junit4] 2> 2431305 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[26]} 0 5
[junit4:junit4] 2> 2431307 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[27 (1436588676161208320)]} 0 0
[junit4:junit4] 2> 2431313 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[27 (1436588676165402624)]} 0 0
[junit4:junit4] 2> 2431313 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[27 (1436588676165402624)]} 0 2
[junit4:junit4] 2> 2431314 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[27]} 0 5
[junit4:junit4] 2> 2431316 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[28 (1436588676170645504)]} 0 0
[junit4:junit4] 2> 2431320 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[28 (1436588676172742656)]} 0 0
[junit4:junit4] 2> 2431321 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[28 (1436588676172742656)]} 0 3
[junit4:junit4] 2> 2431323 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[29 (1436588676177985536)]} 0 0
[junit4:junit4] 2> 2431327 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[29 (1436588676180082688)]} 0 0
[junit4:junit4] 2> 2431327 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[29 (1436588676180082688)]} 0 2
[junit4:junit4] 2> 2431330 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[30 (1436588676185325568)]} 0 0
[junit4:junit4] 2> 2431336 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[30 (1436588676189519872)]} 0 0
[junit4:junit4] 2> 2431336 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[30 (1436588676189519872)]} 0 2
[junit4:junit4] 2> 2431337 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[30]} 0 5
[junit4:junit4] 2> 2431339 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[31 (1436588676194762752)]} 0 0
[junit4:junit4] 2> 2431345 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[31 (1436588676198957056)]} 0 0
[junit4:junit4] 2> 2431345 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[31 (1436588676198957056)]} 0 2
[junit4:junit4] 2> 2431346 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[31]} 0 5
[junit4:junit4] 2> 2431348 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[32 (1436588676204199936)]} 0 0
[junit4:junit4] 2> 2431353 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[32 (1436588676207345664)]} 0 0
[junit4:junit4] 2> 2431353 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[32 (1436588676207345664)]} 0 2
[junit4:junit4] 2> 2431356 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[33 (1436588676212588544)]} 0 0
[junit4:junit4] 2> 2431362 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[33 (1436588676216782848)]} 0 0
[junit4:junit4] 2> 2431362 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[33 (1436588676216782848)]} 0 2
[junit4:junit4] 2> 2431363 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[33]} 0 5
[junit4:junit4] 2> 2431365 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[34 (1436588676222025728)]} 0 0
[junit4:junit4] 2> 2431369 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[34 (1436588676224122880)]} 0 0
[junit4:junit4] 2> 2431369 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[34 (1436588676224122880)]} 0 2
[junit4:junit4] 2> 2431372 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[35 (1436588676229365760)]} 0 0
[junit4:junit4] 2> 2431376 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[35 (1436588676231462912)]} 0 0
[junit4:junit4] 2> 2431376 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[35 (1436588676231462912)]} 0 2
[junit4:junit4] 2> 2431379 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[36 (1436588676236705792)]} 0 0
[junit4:junit4] 2> 2431384 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[36 (1436588676239851520)]} 0 0
[junit4:junit4] 2> 2431384 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[36 (1436588676239851520)]} 0 2
[junit4:junit4] 2> 2431387 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[37 (1436588676245094400)]} 0 0
[junit4:junit4] 2> 2431391 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[37 (1436588676247191552)]} 0 0
[junit4:junit4] 2> 2431391 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[37 (1436588676247191552)]} 0 2
[junit4:junit4] 2> 2431394 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[38 (1436588676252434432)]} 0 0
[junit4:junit4] 2> 2431400 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[38 (1436588676256628736)]} 0 0
[junit4:junit4] 2> 2431400 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[38 (1436588676256628736)]} 0 2
[junit4:junit4] 2> 2431401 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[38]} 0 5
[junit4:junit4] 2> 2431403 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[39 (1436588676261871616)]} 0 0
[junit4:junit4] 2> 2431409 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[39 (1436588676266065920)]} 0 0
[junit4:junit4] 2> 2431409 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[39 (1436588676266065920)]} 0 2
[junit4:junit4] 2> 2431410 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[39]} 0 5
[junit4:junit4] 2> 2431412 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[40 (1436588676271308800)]} 0 0
[junit4:junit4] 2> 2431416 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[40 (1436588676273405952)]} 0 0
[junit4:junit4] 2> 2431417 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[40 (1436588676273405952)]} 0 3
[junit4:junit4] 2> 2431420 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[41 (1436588676279697408)]} 0 0
[junit4:junit4] 2> 2431427 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[41 (1436588676284940288)]} 0 0
[junit4:junit4] 2> 2431427 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[41 (1436588676284940288)]} 0 2
[junit4:junit4] 2> 2431428 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[41]} 0 5
[junit4:junit4] 2> 2431430 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[42 (1436588676290183168)]} 0 0
[junit4:junit4] 2> 2431436 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[42 (1436588676294377472)]} 0 0
[junit4:junit4] 2> 2431436 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[42 (1436588676294377472)]} 0 2
[junit4:junit4] 2> 2431437 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[42]} 0 5
[junit4:junit4] 2> 2431439 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[43 (1436588676299620352)]} 0 0
[junit4:junit4] 2> 2431445 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[43 (1436588676303814656)]} 0 0
[junit4:junit4] 2> 2431445 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[43 (1436588676303814656)]} 0 2
[junit4:junit4] 2> 2431446 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[43]} 0 5
[junit4:junit4] 2> 2431448 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[44 (1436588676309057536)]} 0 0
[junit4:junit4] 2> 2431454 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[44 (1436588676313251840)]} 0 0
[junit4:junit4] 2> 2431454 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[44 (1436588676313251840)]} 0 2
[junit4:junit4] 2> 2431455 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[44]} 0 5
[junit4:junit4] 2> 2431457 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[45 (1436588676318494720)]} 0 0
[junit4:junit4] 2> 2431461 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[45 (1436588676320591872)]} 0 0
[junit4:junit4] 2> 2431461 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[45 (1436588676320591872)]} 0 2
[junit4:junit4] 2> 2431464 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[46 (1436588676325834752)]} 0 0
[junit4:junit4] 2> 2431468 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[46 (1436588676327931904)]} 0 0
[junit4:junit4] 2> 2431468 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[46 (1436588676327931904)]} 0 2
[junit4:junit4] 2> 2431471 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[47 (1436588676333174784)]} 0 0
[junit4:junit4] 2> 2431475 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[47 (1436588676335271936)]} 0 0
[junit4:junit4] 2> 2431475 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[47 (1436588676335271936)]} 0 2
[junit4:junit4] 2> 2431478 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[48 (1436588676340514816)]} 0 0
[junit4:junit4] 2> 2431482 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[48 (1436588676342611968)]} 0 0
[junit4:junit4] 2> 2431482 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[48 (1436588676342611968)]} 0 2
[junit4:junit4] 2> 2431485 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[49 (1436588676347854848)]} 0 0
[junit4:junit4] 2> 2431491 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[49 (1436588676352049152)]} 0 0
[junit4:junit4] 2> 2431491 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[49 (1436588676352049152)]} 0 2
[junit4:junit4] 2> 2431492 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[49]} 0 5
[junit4:junit4] 2> 2431494 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[50 (1436588676357292032)]} 0 0
[junit4:junit4] 2> 2431500 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[50 (1436588676361486336)]} 0 0
[junit4:junit4] 2> 2431500 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[50 (1436588676361486336)]} 0 2
[junit4:junit4] 2> 2431501 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[50]} 0 5
[junit4:junit4] 2> 2431503 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[51 (1436588676366729216)]} 0 0
[junit4:junit4] 2> 2431510 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[51 (1436588676371972096)]} 0 0
[junit4:junit4] 2> 2431510 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[51 (1436588676371972096)]} 0 2
[junit4:junit4] 2> 2431511 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[51]} 0 5
[junit4:junit4] 2> 2431513 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[52 (1436588676377214976)]} 0 0
[junit4:junit4] 2> 2431520 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[52 (1436588676382457856)]} 0 0
[junit4:junit4] 2> 2431520 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[52 (1436588676382457856)]} 0 2
[junit4:junit4] 2> 2431521 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[52]} 0 5
[junit4:junit4] 2> 2431523 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[53 (1436588676387700736)]} 0 0
[junit4:junit4] 2> 2431527 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[53 (1436588676389797888)]} 0 0
[junit4:junit4] 2> 2431527 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[53 (1436588676389797888)]} 0 2
[junit4:junit4] 2> 2431530 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[54 (1436588676395040768)]} 0 0
[junit4:junit4] 2> 2431534 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[54 (1436588676397137920)]} 0 0
[junit4:junit4] 2> 2431534 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[54 (1436588676397137920)]} 0 2
[junit4:junit4] 2> 2431537 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[55 (1436588676402380800)]} 0 0
[junit4:junit4] 2> 2431544 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[55 (1436588676406575104)]} 0 1
[junit4:junit4] 2> 2431544 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[55 (1436588676406575104)]} 0 3
[junit4:junit4] 2> 2431545 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[55]} 0 6
[junit4:junit4] 2> 2431547 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[56 (1436588676412866560)]} 0 0
[junit4:junit4] 2> 2431553 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[56 (1436588676417060864)]} 0 0
[junit4:junit4] 2> 2431553 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[56 (1436588676417060864)]} 0 2
[junit4:junit4] 2> 2431554 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[56]} 0 5
[junit4:junit4] 2> 2431556 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[57 (1436588676422303744)]} 0 0
[junit4:junit4] 2> 2431560 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[57 (1436588676424400896)]} 0 0
[junit4:junit4] 2> 2431560 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[57 (1436588676424400896)]} 0 2
[junit4:junit4] 2> 2431563 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[58 (1436588676429643776)]} 0 0
[junit4:junit4] 2> 2431569 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[58 (1436588676433838080)]} 0 0
[junit4:junit4] 2> 2431570 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[58 (1436588676433838080)]} 0 3
[junit4:junit4] 2> 2431570 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[58]} 0 5
[junit4:junit4] 2> 2431572 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[59 (1436588676439080960)]} 0 0
[junit4:junit4] 2> 2431576 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[59 (1436588676441178112)]} 0 0
[junit4:junit4] 2> 2431576 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[59 (1436588676441178112)]} 0 2
[junit4:junit4] 2> 2431579 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[60 (1436588676446420992)]} 0 0
[junit4:junit4] 2> 2431585 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[60 (1436588676450615296)]} 0 0
[junit4:junit4] 2> 2431586 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[60 (1436588676450615296)]} 0 2
[junit4:junit4] 2> 2431586 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[60]} 0 5
[junit4:junit4] 2> 2431588 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[61 (1436588676455858176)]} 0 0
[junit4:junit4] 2> 2431594 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[61 (1436588676460052480)]} 0 0
[junit4:junit4] 2> 2431594 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[61 (1436588676460052480)]} 0 2
[junit4:junit4] 2> 2431595 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[61]} 0 5
[junit4:junit4] 2> 2431597 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[62 (1436588676465295360)]} 0 0
[junit4:junit4] 2> 2431603 T7830 C22596 P54620 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[62 (1436588676469489664)]} 0 0
[junit4:junit4] 2> 2431603 T7799 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[62 (1436588676469489664)]} 0 2
[junit4:junit4] 2> 2431604 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[62]} 0 5
[junit4:junit4] 2> 2431606 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[63 (1436588676474732544)]} 0 0
[junit4:junit4] 2> 2431610 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[63 (1436588676476829696)]} 0 0
[junit4:junit4] 2> 2431610 T7783 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[63 (1436588676476829696)]} 0 2
[junit4:junit4] 2> 2431613 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[64 (1436588676482072576)]} 0 0
[junit4:junit4] 2> 2431619 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[64 (1436588676486266880)]} 0 0
[junit4:junit4] 2> 2431619 T7785 C22598 P54610 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54613/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[64 (1436588676486266880)]} 0 2
[junit4:junit4] 2> 2431620 T7801 C22599 P54613 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[64]} 0 5
[junit4:junit4] 2> 2431622 T7759 C22600 P54603 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[65 (1436588676491509760)]} 0 0
[junit4:junit4] 2> 2431626 T7811 C22597 P54616 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54610/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[65 (1436588676493606912)]}
[...truncated too long message...]
53902 T7747 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
[junit4:junit4] 2> 2753902 T7747 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
[junit4:junit4] 2> 2753903 T7747 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
[junit4:junit4] 2> 2753905 T7747 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
[junit4:junit4] 2> 2753905 T7747 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
[junit4:junit4] 2> 2753906 T7747 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index;done=false>>]
[junit4:junit4] 2> 2753906 T7747 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4/index
[junit4:junit4] 2> 2753919 T7747 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4;done=false>>]
[junit4:junit4] 2> 2753920 T7747 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154/jetty4
[junit4:junit4] 2> 2753921 T7902 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=89786790521143308-127.0.0.1:54620_-n_0000000004) am no longer a leader.
[junit4:junit4] 2> 2753922 T7838 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 2753922 T7838 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 2753922 T7747 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
[junit4:junit4] 2> 2753923 T7838 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 2753993 T7747 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 2753995 T7747 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:54600 54600
[junit4:junit4] 2> 2754141 T7821 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 2754177 T7821 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 2754178 T7821 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 2754179 T7747 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 54603
[junit4:junit4] 2> !!!! WARNING: best effort to remove /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1370037697154 FAILED !!!!!
[junit4:junit4] 2> 2754180 T7747 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 2754181 T7747 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:54600 54600
[junit4:junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4:junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeyShardSplitTest -Dtests.method=testDistribSearch -Dtests.seed=D5E6A581AA59CC86 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt -Dtests.locale=sk -Dtests.timezone=Africa/Kigali -Dtests.file.encoding=ISO-8859-1
[junit4:junit4] ERROR 349s J1 | ChaosMonkeyShardSplitTest.testDistribSearch <<<
[junit4:junit4] > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:54610 returned non ok status:500, message:Server Error
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([D5E6A581AA59CC86:54002B99DD06ACBA]:0)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4:junit4] > at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:208)
[junit4:junit4] > at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:136)
[junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:679)
[junit4:junit4] 2> 2754200 T7747 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4] 2> 349442 T7746 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
[junit4:junit4] 2> NOTE: test params are: codec=Lucene41, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=sk, timezone=Africa/Kigali
[junit4:junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Sun Microsystems Inc. 1.6.0_32 (64-bit)/cpus=16,threads=1,free=315277320,total=518782976
[junit4:junit4] 2> NOTE: All tests run in this JVM: [TestRangeQuery, TestSuggestSpellingConverter, TestStressReorder, XsltUpdateRequestHandlerTest, SignatureUpdateProcessorFactoryTest, TestPropInject, StatsComponentTest, SpellCheckComponentTest, TestWordDelimiterFilterFactory, PingRequestHandlerTest, TestOmitPositions, IndexReaderFactoryTest, ZkNodePropsTest, TestCollationField, SampleTest, TestJmxMonitoredMap, SolrCmdDistributorTest, TestFastOutputStream, TestMultiCoreConfBootstrap, FullSolrCloudDistribCmdsTest, SuggesterWFSTTest, SimplePostToolTest, TestLuceneMatchVersion, CachingDirectoryFactoryTest, TestSolrQueryParserResource, TestCodecSupport, HighlighterTest, TestReload, OutputWriterTest, TestCopyFieldCollectionResource, BasicZkTest, ZkSolrClientTest, TestBinaryResponseWriter, TestRTGBase, FileUtilsTest, TestUniqueKeyFieldResource, TestSolrQueryParserDefaultOperatorResource, SolrTestCaseJ4Test, TestFoldingMultitermQuery, TestCSVLoader, TestFiltering, TestDefaultSimilarityFactory, EchoParamsTest, TestSchemaNameResource, TestSystemIdResolver, TestZkChroot, TestRecovery, SortByFunctionTest, NoCacheHeaderTest, TestDistributedGrouping, ConvertedLegacyTest, TestPartialUpdateDeduplication, HighlighterConfigTest, TestNumberUtils, TestIndexSearcher, PathHierarchyTokenizerFactoryTest, JSONWriterTest, ChaosMonkeySafeLeaderTest, TermVectorComponentDistributedTest, URLClassifyProcessorTest, TestSolrXml, TestJmxIntegration, MultiTermTest, TestElisionMultitermQuery, DebugComponentTest, TestDefaultSearchFieldResource, TestIndexingPerformance, DocValuesMultiTest, NotRequiredUniqueKeyTest, SolrPluginUtilsTest, TestAtomicUpdateErrorCases, BasicDistributedZkTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, ZkCLITest, ShardRoutingCustomTest, TestDistributedSearch, TestHashPartitioner, DistributedSpellCheckComponentTest, ZkControllerTest, TestStressVersions, HardAutoCommitTest, DistributedTermsComponentTest, TestCoreContainer, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, PeerSyncTest, BadIndexSchemaTest, TestSort, BasicFunctionalityTest, TestLazyCores, DirectUpdateHandlerTest, TestBadConfig, OverseerCollectionProcessorTest, SoftAutoCommitTest, CurrencyFieldOpenExchangeTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, TestCoreDiscovery, TestExtendedDismaxParser, SpellCheckCollatorTest, SuggesterFSTTest, SolrRequestParserTest, DocValuesTest, SuggesterTSTTest, SuggesterTest, TestStressLucene, SpatialFilterTest, PolyFieldTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestUpdate, DefaultValueUpdateProcessorTest, TestRemoteStreaming, DistanceFunctionTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, TestQueryUtils, DirectSolrSpellCheckerTest, TestQueryTypes, XmlUpdateRequestHandlerTest, TestValueSourceCache, RequiredFieldsTest, TestArbitraryIndexDir, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, SearchHandlerTest, TestQuerySenderListener, UpdateParamsTest, TestQuerySenderNoQuery, ResponseLogComponentTest, BadComponentTest, TestStressRecovery, ChaosMonkeyShardSplitTest]
[junit4:junit4] Completed on J1 in 349.96s, 1 test, 1 error <<< FAILURES!
[...truncated 10 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:393: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:366: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:181: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:437: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1243: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:887: There were test failures: 296 suites, 1236 tests, 1 error, 12 ignored (6 assumptions)
Total time: 164 minutes 30 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 272 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/272/
1 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeyShardSplitTest.testDistribSearch
Error Message:
Server at http://127.0.0.1:17335 returned non ok status:500, message:Server Error
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:17335 returned non ok status:500, message:Server Error
at __randomizedtesting.SeedInfo.seed([584A30134BDBB210:D9ACBE0B3C84D22C]:0)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:208)
at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:136)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:679)
Build Log:
[...truncated 8897 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeyShardSplitTest
[junit4:junit4] 2> 276900 T506 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
[junit4:junit4] 2> 276906 T506 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solrtest-ChaosMonkeyShardSplitTest-1369957632535
[junit4:junit4] 2> 276907 T506 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 276908 T507 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 277008 T506 oasc.ZkTestServer.run start zk server on port:27818
[junit4:junit4] 2> 277008 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 277096 T513 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11d8e635 name:ZooKeeperConnection Watcher:127.0.0.1:27818 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 277096 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 277097 T506 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 277183 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 277216 T515 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c4eeac0 name:ZooKeeperConnection Watcher:127.0.0.1:27818/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 277216 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 277217 T506 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 277280 T506 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 277306 T506 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 277342 T506 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 277345 T506 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 277346 T506 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 277353 T506 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 277354 T506 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 277468 T506 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 277468 T506 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 277476 T506 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 277477 T506 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 277487 T506 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 277487 T506 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 277495 T506 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 277495 T506 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 277499 T506 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 277500 T506 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 277504 T506 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 277505 T506 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 277509 T506 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 277510 T506 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 277813 T506 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 277817 T506 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27471
[junit4:junit4] 2> 277818 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 277818 T506 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 277819 T506 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369957633144
[junit4:junit4] 2> 277819 T506 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369957633144/solr.xml
[junit4:junit4] 2> 277820 T506 oasc.CoreContainer.<init> New CoreContainer 1258062220
[junit4:junit4] 2> 277821 T506 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369957633144/'
[junit4:junit4] 2> 277821 T506 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369957633144/'
[junit4:junit4] 2> 277937 T506 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 277938 T506 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 277939 T506 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 277939 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 277940 T506 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 277940 T506 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 277941 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 277941 T506 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 277942 T506 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 277942 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 277960 T506 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 277961 T506 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27818/solr
[junit4:junit4] 2> 277962 T506 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 277963 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 277966 T526 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ad0db19 name:ZooKeeperConnection Watcher:127.0.0.1:27818 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 277967 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 277970 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 277983 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 277985 T528 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14b1666d name:ZooKeeperConnection Watcher:127.0.0.1:27818/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 277985 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 277988 T506 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 277998 T506 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 278002 T506 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 278005 T506 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27471_
[junit4:junit4] 2> 278006 T506 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27471_
[junit4:junit4] 2> 278015 T506 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 278021 T506 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 278024 T506 oasc.Overseer.start Overseer (id=89781543406272515-127.0.0.1:27471_-n_0000000000) starting
[junit4:junit4] 2> 278029 T506 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 278034 T530 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 278035 T506 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 278038 T506 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 278041 T506 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 278044 T529 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 278048 T531 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369957633144/collection1
[junit4:junit4] 2> 278048 T531 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 278049 T531 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 278049 T531 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 278051 T531 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369957633144/collection1/'
[junit4:junit4] 2> 278053 T531 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369957633144/collection1/lib/README' to classloader
[junit4:junit4] 2> 278054 T531 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369957633144/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 278105 T531 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 278164 T531 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 278266 T531 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 278273 T531 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 278846 T531 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 278850 T531 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 278853 T531 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 278858 T531 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 278891 T531 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 278892 T531 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369957633144/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/control/data/
[junit4:junit4] 2> 278892 T531 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e848ecc
[junit4:junit4] 2> 278893 T531 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 278894 T531 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/control/data
[junit4:junit4] 2> 278894 T531 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/control/data/index/
[junit4:junit4] 2> 278894 T531 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 278895 T531 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/control/data/index
[junit4:junit4] 2> 278899 T531 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1f002d1f lockFactory=org.apache.lucene.store.NativeFSLockFactory@5e0cb13b),segFN=segments_1,generation=1}
[junit4:junit4] 2> 278900 T531 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 278903 T531 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 278903 T531 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 278904 T531 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 278904 T531 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 278905 T531 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 278905 T531 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 278906 T531 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 278906 T531 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 278907 T531 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 278916 T531 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 278924 T531 oass.SolrIndexSearcher.<init> Opening Searcher@ae7b92e main
[junit4:junit4] 2> 278924 T531 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/control/data/tlog
[junit4:junit4] 2> 278926 T531 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 278926 T531 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 278929 T532 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ae7b92e main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 278931 T531 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 278932 T531 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 279549 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 279550 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:27471_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:27471"}
[junit4:junit4] 2> 279550 T529 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 279551 T529 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 279568 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 279937 T531 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 279937 T531 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27471 collection:control_collection shard:shard1
[junit4:junit4] 2> 279938 T531 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 279956 T531 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 279968 T531 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 279969 T531 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 279969 T531 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27471/collection1/
[junit4:junit4] 2> 279969 T531 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 279970 T531 oasc.SyncStrategy.syncToMe http://127.0.0.1:27471/collection1/ has no replicas
[junit4:junit4] 2> 279970 T531 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:27471/collection1/
[junit4:junit4] 2> 279970 T531 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 281074 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 281106 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 281156 T531 oasc.ZkController.register We are http://127.0.0.1:27471/collection1/ and leader is http://127.0.0.1:27471/collection1/
[junit4:junit4] 2> 281156 T531 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:27471
[junit4:junit4] 2> 281156 T531 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 281156 T531 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 281157 T531 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 281159 T531 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 281160 T506 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 281161 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 281161 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 281178 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 281180 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 281182 T535 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@327ec4d0 name:ZooKeeperConnection Watcher:127.0.0.1:27818/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 281182 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 281184 T506 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 281187 T506 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 281357 T506 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 281360 T506 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:17335
[junit4:junit4] 2> 281361 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 281361 T506 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 281361 T506 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369957636821
[junit4:junit4] 2> 281362 T506 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369957636821/solr.xml
[junit4:junit4] 2> 281362 T506 oasc.CoreContainer.<init> New CoreContainer 1432043681
[junit4:junit4] 2> 281363 T506 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369957636821/'
[junit4:junit4] 2> 281363 T506 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369957636821/'
[junit4:junit4] 2> 281438 T506 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 281439 T506 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 281439 T506 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 281439 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 281440 T506 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 281440 T506 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 281440 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 281441 T506 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 281441 T506 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 281442 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 281452 T506 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 281453 T506 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27818/solr
[junit4:junit4] 2> 281453 T506 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 281454 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 281457 T546 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1afb3ad9 name:ZooKeeperConnection Watcher:127.0.0.1:27818 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 281457 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 281467 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 281479 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 281481 T548 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3cc81bd9 name:ZooKeeperConnection Watcher:127.0.0.1:27818/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 281481 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 281491 T506 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 282495 T506 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:17335_
[junit4:junit4] 2> 282500 T506 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:17335_
[junit4:junit4] 2> 282504 T528 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 282504 T535 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 282504 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 282504 T548 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 282520 T549 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369957636821/collection1
[junit4:junit4] 2> 282520 T549 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 282521 T549 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 282521 T549 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 282523 T549 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369957636821/collection1/'
[junit4:junit4] 2> 282524 T549 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369957636821/collection1/lib/README' to classloader
[junit4:junit4] 2> 282525 T549 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369957636821/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 282578 T549 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 282622 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 282623 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:27471__collection1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:27471_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:27471"}
[junit4:junit4] 2> 282635 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 282635 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 282635 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 282637 T549 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 282739 T549 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 282745 T549 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 283330 T549 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 283335 T549 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 283337 T549 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 283343 T549 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 283375 T549 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 283375 T549 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369957636821/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty1/
[junit4:junit4] 2> 283375 T549 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e848ecc
[junit4:junit4] 2> 283376 T549 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 283377 T549 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty1
[junit4:junit4] 2> 283377 T549 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty1/index/
[junit4:junit4] 2> 283377 T549 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 283378 T549 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty1/index
[junit4:junit4] 2> 283382 T549 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@651b289f lockFactory=org.apache.lucene.store.NativeFSLockFactory@695eb69a),segFN=segments_1,generation=1}
[junit4:junit4] 2> 283382 T549 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 283385 T549 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 283385 T549 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 283386 T549 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 283387 T549 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 283388 T549 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 283388 T549 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 283388 T549 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 283389 T549 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 283390 T549 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 283399 T549 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 283407 T549 oass.SolrIndexSearcher.<init> Opening Searcher@204cb73c main
[junit4:junit4] 2> 283407 T549 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty1/tlog
[junit4:junit4] 2> 283408 T549 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 283408 T549 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 283414 T550 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@204cb73c main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 283416 T549 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 283417 T549 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 284141 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 284142 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:17335_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:17335"}
[junit4:junit4] 2> 284142 T529 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4:junit4] 2> 284143 T529 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 284159 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 284159 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 284159 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 284421 T549 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 284421 T549 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:17335 collection:collection1 shard:shard1
[junit4:junit4] 2> 284422 T549 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 284443 T549 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 284447 T549 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 284447 T549 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 284447 T549 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:17335/collection1/
[junit4:junit4] 2> 284448 T549 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 284448 T549 oasc.SyncStrategy.syncToMe http://127.0.0.1:17335/collection1/ has no replicas
[junit4:junit4] 2> 284448 T549 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:17335/collection1/
[junit4:junit4] 2> 284448 T549 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 285665 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 285690 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 285690 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 285690 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 285735 T549 oasc.ZkController.register We are http://127.0.0.1:17335/collection1/ and leader is http://127.0.0.1:17335/collection1/
[junit4:junit4] 2> 285735 T549 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:17335
[junit4:junit4] 2> 285735 T549 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 285735 T549 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 285736 T549 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 285738 T549 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 285739 T506 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 285740 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 285740 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 286015 T506 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 286018 T506 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39715
[junit4:junit4] 2> 286019 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 286019 T506 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 286020 T506 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369957641385
[junit4:junit4] 2> 286020 T506 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369957641385/solr.xml
[junit4:junit4] 2> 286021 T506 oasc.CoreContainer.<init> New CoreContainer 364333457
[junit4:junit4] 2> 286021 T506 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369957641385/'
[junit4:junit4] 2> 286022 T506 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369957641385/'
[junit4:junit4] 2> 286133 T506 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 286134 T506 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 286134 T506 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 286135 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 286135 T506 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 286136 T506 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 286136 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 286137 T506 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 286137 T506 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 286138 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 286154 T506 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 286155 T506 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27818/solr
[junit4:junit4] 2> 286155 T506 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 286157 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 286160 T562 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a485d22 name:ZooKeeperConnection Watcher:127.0.0.1:27818 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 286160 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 286173 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 286184 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 286187 T564 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e414fa5 name:ZooKeeperConnection Watcher:127.0.0.1:27818/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 286187 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 286198 T506 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 287197 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 287198 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:17335__collection1",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:17335_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:17335"}
[junit4:junit4] 2> 287202 T506 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39715_
[junit4:junit4] 2> 287239 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 287239 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 287239 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 287239 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 287240 T506 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39715_
[junit4:junit4] 2> 287258 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 287258 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 287259 T535 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 287259 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 287259 T528 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 287260 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 287260 T548 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 287261 T564 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 287268 T565 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369957641385/collection1
[junit4:junit4] 2> 287268 T565 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 287269 T565 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 287269 T565 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 287271 T565 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369957641385/collection1/'
[junit4:junit4] 2> 287273 T565 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369957641385/collection1/lib/README' to classloader
[junit4:junit4] 2> 287274 T565 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369957641385/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 287332 T565 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 287397 T565 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 287499 T565 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 287506 T565 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 288100 T565 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 288108 T565 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 288110 T565 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 288116 T565 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 288146 T565 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 288146 T565 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369957641385/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty2/
[junit4:junit4] 2> 288147 T565 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e848ecc
[junit4:junit4] 2> 288147 T565 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 288148 T565 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty2
[junit4:junit4] 2> 288148 T565 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty2/index/
[junit4:junit4] 2> 288148 T565 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 288149 T565 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty2/index
[junit4:junit4] 2> 288153 T565 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32ca113e lockFactory=org.apache.lucene.store.NativeFSLockFactory@a90b3f7),segFN=segments_1,generation=1}
[junit4:junit4] 2> 288153 T565 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 288156 T565 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 288157 T565 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 288157 T565 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 288158 T565 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 288159 T565 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 288159 T565 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 288160 T565 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 288160 T565 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 288161 T565 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 288170 T565 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 288178 T565 oass.SolrIndexSearcher.<init> Opening Searcher@4b5a82f7 main
[junit4:junit4] 2> 288178 T565 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty2/tlog
[junit4:junit4] 2> 288179 T565 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 288179 T565 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 288184 T566 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4b5a82f7 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 288187 T565 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 288187 T565 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 288754 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 288755 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:39715_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:39715"}
[junit4:junit4] 2> 288756 T529 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 288756 T529 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 288770 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 288770 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 288770 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 288770 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 289190 T565 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 289190 T565 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39715 collection:collection1 shard:shard2
[junit4:junit4] 2> 289191 T565 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 289207 T565 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 289210 T565 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 289211 T565 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 289211 T565 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39715/collection1/
[junit4:junit4] 2> 289211 T565 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 289212 T565 oasc.SyncStrategy.syncToMe http://127.0.0.1:39715/collection1/ has no replicas
[junit4:junit4] 2> 289212 T565 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39715/collection1/
[junit4:junit4] 2> 289212 T565 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 290276 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 290319 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 290319 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 290319 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 290319 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 290347 T565 oasc.ZkController.register We are http://127.0.0.1:39715/collection1/ and leader is http://127.0.0.1:39715/collection1/
[junit4:junit4] 2> 290347 T565 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39715
[junit4:junit4] 2> 290347 T565 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 290347 T565 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 290348 T565 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 290350 T565 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 290351 T506 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 290352 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 290352 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 290621 T506 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 290624 T506 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:12138
[junit4:junit4] 2> 290625 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 290626 T506 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 290626 T506 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369957645997
[junit4:junit4] 2> 290627 T506 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369957645997/solr.xml
[junit4:junit4] 2> 290627 T506 oasc.CoreContainer.<init> New CoreContainer 1988913297
[junit4:junit4] 2> 290628 T506 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369957645997/'
[junit4:junit4] 2> 290628 T506 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369957645997/'
[junit4:junit4] 2> 290740 T506 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 290740 T506 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 290741 T506 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 290741 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 290742 T506 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 290742 T506 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 290743 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 290743 T506 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 290744 T506 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 290744 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 290762 T506 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 290763 T506 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27818/solr
[junit4:junit4] 2> 290764 T506 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 290765 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 290768 T578 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72ad40e8 name:ZooKeeperConnection Watcher:127.0.0.1:27818 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 290768 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 290779 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 290792 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 290794 T580 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2aad9064 name:ZooKeeperConnection Watcher:127.0.0.1:27818/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 290794 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 290807 T506 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 291811 T506 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:12138_
[junit4:junit4] 2> 291814 T506 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:12138_
[junit4:junit4] 2> 291871 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 291872 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 291873 T528 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 291873 T580 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 291873 T535 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 291873 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 291874 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 291874 T548 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 291875 T564 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 291876 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 291877 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:39715__collection1",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:39715_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:39715"}
[junit4:junit4] 2> 291881 T581 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369957645997/collection1
[junit4:junit4] 2> 291882 T581 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 291889 T581 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 291890 T581 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 291932 T581 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369957645997/collection1/'
[junit4:junit4] 2> 291934 T581 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369957645997/collection1/lib/README' to classloader
[junit4:junit4] 2> 291934 T580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 291934 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 291934 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 291934 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 291934 T581 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369957645997/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 291938 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 291987 T581 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 292051 T581 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 292153 T581 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 292159 T581 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 292728 T581 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 292733 T581 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 292735 T581 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 292740 T581 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 292770 T581 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 292770 T581 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369957645997/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/
[junit4:junit4] 2> 292770 T581 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e848ecc
[junit4:junit4] 2> 292771 T581 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 292772 T581 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3
[junit4:junit4] 2> 292772 T581 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/index/
[junit4:junit4] 2> 292772 T581 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 292773 T581 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/index
[junit4:junit4] 2> 292777 T581 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32c98179 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3ab0d58),segFN=segments_1,generation=1}
[junit4:junit4] 2> 292777 T581 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 292781 T581 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 292781 T581 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 292782 T581 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 292783 T581 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 292783 T581 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 292784 T581 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 292784 T581 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 292785 T581 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 292785 T581 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 292795 T581 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 292802 T581 oass.SolrIndexSearcher.<init> Opening Searcher@547fcdfc main
[junit4:junit4] 2> 292803 T581 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/tlog
[junit4:junit4] 2> 292804 T581 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 292804 T581 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 292810 T582 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@547fcdfc main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 292812 T581 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 292812 T581 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 293440 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 293441 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:12138_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:12138"}
[junit4:junit4] 2> 293441 T529 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 293442 T529 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 293459 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 293459 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 293459 T580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 293459 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 293459 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 293817 T581 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 293817 T581 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:12138 collection:collection1 shard:shard1
[junit4:junit4] 2> 293821 T581 oasc.ZkController.register We are http://127.0.0.1:12138/collection1/ and leader is http://127.0.0.1:17335/collection1/
[junit4:junit4] 2> 293821 T581 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:12138
[junit4:junit4] 2> 293822 T581 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 293822 T581 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C138 name=collection1 org.apache.solr.core.SolrCore@5bd7d30f url=http://127.0.0.1:12138/collection1 node=127.0.0.1:12138_ C138_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:12138_, base_url=http://127.0.0.1:12138}
[junit4:junit4] 2> 293822 T583 C138 P12138 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 293823 T581 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 293823 T583 C138 P12138 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 293824 T583 C138 P12138 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 293824 T583 C138 P12138 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 293824 T506 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 293825 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 293826 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 293826 T583 C138 P12138 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 293852 T538 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 127.0.0.1:12138__collection1, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 294110 T506 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 294112 T506 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:24293
[junit4:junit4] 2> 294113 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 294114 T506 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 294114 T506 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369957649478
[junit4:junit4] 2> 294115 T506 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369957649478/solr.xml
[junit4:junit4] 2> 294115 T506 oasc.CoreContainer.<init> New CoreContainer 1840278909
[junit4:junit4] 2> 294116 T506 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369957649478/'
[junit4:junit4] 2> 294117 T506 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369957649478/'
[junit4:junit4] 2> 294228 T506 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 294229 T506 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 294230 T506 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 294230 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 294231 T506 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 294231 T506 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 294232 T506 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 294232 T506 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 294233 T506 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 294233 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 294250 T506 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 294251 T506 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:27818/solr
[junit4:junit4] 2> 294251 T506 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 294252 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 294258 T595 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20d107d6 name:ZooKeeperConnection Watcher:127.0.0.1:27818 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 294259 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 294276 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 294290 T506 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 294298 T597 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6bf1640f name:ZooKeeperConnection Watcher:127.0.0.1:27818/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 294298 T506 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 294304 T506 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 294965 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 294966 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:12138__collection1",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:12138_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:12138"}
[junit4:junit4] 2> 294984 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 294984 T580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 294984 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 294984 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 294984 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 294984 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 295308 T506 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:24293_
[junit4:junit4] 2> 295310 T506 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:24293_
[junit4:junit4] 2> 295313 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 295313 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 295313 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 295313 T535 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 295313 T580 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 295313 T528 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 295314 T580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 295314 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 295314 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 295315 T548 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 295315 T564 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 295315 T597 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 295327 T598 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369957649478/collection1
[junit4:junit4] 2> 295327 T598 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 295328 T598 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 295328 T598 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 295330 T598 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369957649478/collection1/'
[junit4:junit4] 2> 295332 T598 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369957649478/collection1/lib/README' to classloader
[junit4:junit4] 2> 295332 T598 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369957649478/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 295388 T598 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 295447 T598 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 295548 T598 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 295555 T598 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 295854 T538 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 127.0.0.1:12138__collection1, state: recovering, checkLive: true, onlyIfLeader: true for: 2 seconds.
[junit4:junit4] 2> 295854 T538 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=127.0.0.1:12138__collection1&state=recovering&nodeName=127.0.0.1:12138_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=2002
[junit4:junit4] 2> 296207 T598 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 296211 T598 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 296214 T598 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 296220 T598 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 296251 T598 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 296252 T598 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369957649478/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/
[junit4:junit4] 2> 296252 T598 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6e848ecc
[junit4:junit4] 2> 296253 T598 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 296253 T598 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4
[junit4:junit4] 2> 296254 T598 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index/
[junit4:junit4] 2> 296254 T598 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 296255 T598 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index
[junit4:junit4] 2> 296259 T598 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@198d7173 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7da22e4a),segFN=segments_1,generation=1}
[junit4:junit4] 2> 296259 T598 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 296263 T598 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 296263 T598 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 296264 T598 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 296265 T598 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 296266 T598 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 296266 T598 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 296266 T598 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 296267 T598 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 296267 T598 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 296280 T598 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 296289 T598 oass.SolrIndexSearcher.<init> Opening Searcher@3851af4e main
[junit4:junit4] 2> 296290 T598 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/tlog
[junit4:junit4] 2> 296291 T598 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 296291 T598 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 296301 T599 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3851af4e main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 296303 T598 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 296303 T598 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 296490 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 296491 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":null,
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:24293_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:24293"}
[junit4:junit4] 2> 296492 T529 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 296492 T529 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 296504 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 296504 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 296504 T580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 296504 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 296504 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 296504 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 297332 T598 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 297332 T598 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:24293 collection:collection1 shard:shard2
[junit4:junit4] 2> 297337 T598 oasc.ZkController.register We are http://127.0.0.1:24293/collection1/ and leader is http://127.0.0.1:39715/collection1/
[junit4:junit4] 2> 297337 T598 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:24293
[junit4:junit4] 2> 297337 T598 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 297337 T598 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C139 name=collection1 org.apache.solr.core.SolrCore@3ac8119 url=http://127.0.0.1:24293/collection1 node=127.0.0.1:24293_ C139_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:24293_, base_url=http://127.0.0.1:24293}
[junit4:junit4] 2> 297338 T600 C139 P24293 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 297339 T600 C139 P24293 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 297339 T598 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 297339 T600 C139 P24293 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 297340 T600 C139 P24293 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 297340 T506 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 297341 T506 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 297341 T600 C139 P24293 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 297342 T506 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 297365 T506 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 297366 T554 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 127.0.0.1:24293__collection1, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 297367 T506 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 15 for each attempt
[junit4:junit4] 2> 297368 T506 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4:junit4] 2> 297369 T506 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> ASYNC NEW_CORE C140 name=collection1 org.apache.solr.core.SolrCore@5bd7d30f url=http://127.0.0.1:12138/collection1 node=127.0.0.1:12138_ C140_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:12138_, base_url=http://127.0.0.1:12138}
[junit4:junit4] 2> 297856 T583 C140 P12138 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:17335/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 297856 T583 C140 P12138 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:12138 START replicas=[http://127.0.0.1:17335/collection1/] nUpdates=100
[junit4:junit4] 2> 297857 T583 C140 P12138 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 297857 T583 C140 P12138 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 297858 T583 C140 P12138 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 297858 T583 C140 P12138 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 297858 T583 C140 P12138 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 297859 T583 C140 P12138 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:17335/collection1/. core=collection1
[junit4:junit4] 2> 297859 T583 C140 P12138 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C141 name=collection1 org.apache.solr.core.SolrCore@4e87c296 url=http://127.0.0.1:17335/collection1 node=127.0.0.1:17335_ C141_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:17335_, base_url=http://127.0.0.1:17335, leader=true}
[junit4:junit4] 2> 297866 T539 C141 P17335 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 297873 T540 C141 P17335 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 297875 T540 C141 P17335 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@651b289f lockFactory=org.apache.lucene.store.NativeFSLockFactory@695eb69a),segFN=segments_1,generation=1}
[junit4:junit4] 2> 297876 T540 C141 P17335 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 297876 T540 C141 P17335 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@651b289f lockFactory=org.apache.lucene.store.NativeFSLockFactory@695eb69a),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@651b289f lockFactory=org.apache.lucene.store.NativeFSLockFactory@695eb69a),segFN=segments_2,generation=2}
[junit4:junit4] 2> 297877 T540 C141 P17335 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 297877 T540 C141 P17335 oass.SolrIndexSearcher.<init> Opening Searcher@402badd6 realtime
[junit4:junit4] 2> 297878 T540 C141 P17335 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 297878 T540 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 6
[junit4:junit4] 2> 297879 T583 C140 P12138 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 297879 T583 C140 P12138 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 297881 T541 C141 P17335 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 297882 T541 C141 P17335 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=1
[junit4:junit4] 2> 297882 T583 C140 P12138 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 297883 T583 C140 P12138 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 297883 T583 C140 P12138 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 297885 T541 C141 P17335 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 297886 T583 C140 P12138 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 297887 T583 C140 P12138 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/index.20130530194733514
[junit4:junit4] 2> 297887 T583 C140 P12138 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@5d3908f1 lockFactory=org.apache.lucene.store.NativeFSLockFactory@62120c6b) fullCopy=false
[junit4:junit4] 2> 297890 T541 C141 P17335 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 297891 T583 C140 P12138 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 297892 T583 C140 P12138 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 297892 T583 C140 P12138 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 297894 T583 C140 P12138 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32c98179 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3ab0d58),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32c98179 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3ab0d58),segFN=segments_2,generation=2}
[junit4:junit4] 2> 297895 T583 C140 P12138 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 297895 T583 C140 P12138 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 297895 T583 C140 P12138 oass.SolrIndexSearcher.<init> Opening Searcher@40a2b174 main
[junit4:junit4] 2> 297897 T582 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@40a2b174 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 297897 T583 C140 P12138 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/index.20130530194733514 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/index.20130530194733514;done=true>>]
[junit4:junit4] 2> 297898 T583 C140 P12138 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/index.20130530194733514
[junit4:junit4] 2> 297898 T583 C140 P12138 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty3/index.20130530194733514
[junit4:junit4] 2> 297898 T583 C140 P12138 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 297899 T583 C140 P12138 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 297899 T583 C140 P12138 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 297899 T583 C140 P12138 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 297901 T583 C140 P12138 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 298016 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 298017 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:24293__collection1",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:24293_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:24293"}
[junit4:junit4] 2> 298021 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:12138__collection1",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:12138_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:12138"}
[junit4:junit4] 2> 298034 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 298034 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 298034 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 298034 T580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 298034 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 298034 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 298368 T554 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 127.0.0.1:24293__collection1, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 298368 T554 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=127.0.0.1:24293__collection1&state=recovering&nodeName=127.0.0.1:24293_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1002
[junit4:junit4] 2> 298371 T506 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 299374 T506 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> C139_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:24293_, base_url=http://127.0.0.1:24293}
[junit4:junit4] 2> 300370 T600 C139 P24293 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:39715/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 300370 T600 C139 P24293 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:24293 START replicas=[http://127.0.0.1:39715/collection1/] nUpdates=100
[junit4:junit4] 2> 300371 T600 C139 P24293 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 300371 T600 C139 P24293 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 300371 T600 C139 P24293 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 300372 T600 C139 P24293 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 300372 T600 C139 P24293 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 300373 T600 C139 P24293 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:39715/collection1/. core=collection1
[junit4:junit4] 2> 300373 T600 C139 P24293 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 300377 T506 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> ASYNC NEW_CORE C142 name=collection1 org.apache.solr.core.SolrCore@1a5f8113 url=http://127.0.0.1:39715/collection1 node=127.0.0.1:39715_ C142_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:39715_, base_url=http://127.0.0.1:39715, leader=true}
[junit4:junit4] 2> 300380 T555 C142 P39715 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 300385 T556 C142 P39715 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 300388 T556 C142 P39715 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32ca113e lockFactory=org.apache.lucene.store.NativeFSLockFactory@a90b3f7),segFN=segments_1,generation=1}
[junit4:junit4] 2> 300388 T556 C142 P39715 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 300389 T556 C142 P39715 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32ca113e lockFactory=org.apache.lucene.store.NativeFSLockFactory@a90b3f7),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32ca113e lockFactory=org.apache.lucene.store.NativeFSLockFactory@a90b3f7),segFN=segments_2,generation=2}
[junit4:junit4] 2> 300389 T556 C142 P39715 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 300390 T556 C142 P39715 oass.SolrIndexSearcher.<init> Opening Searcher@8a212f9 realtime
[junit4:junit4] 2> 300390 T556 C142 P39715 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 300391 T556 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 6
[junit4:junit4] 2> 300392 T600 C139 P24293 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 300392 T600 C139 P24293 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 300394 T557 C142 P39715 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 300394 T557 C142 P39715 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=1
[junit4:junit4] 2> 300395 T600 C139 P24293 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 300395 T600 C139 P24293 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 300395 T600 C139 P24293 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 300398 T557 C142 P39715 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 300399 T600 C139 P24293 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 300400 T600 C139 P24293 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index.20130530194736027
[junit4:junit4] 2> 300400 T600 C139 P24293 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@7280c708 lockFactory=org.apache.lucene.store.NativeFSLockFactory@321ef9ca) fullCopy=false
[junit4:junit4] 2> 300403 T557 C142 P39715 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 300404 T600 C139 P24293 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 300405 T600 C139 P24293 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 300405 T600 C139 P24293 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 300407 T600 C139 P24293 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@198d7173 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7da22e4a),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@198d7173 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7da22e4a),segFN=segments_2,generation=2}
[junit4:junit4] 2> 300407 T600 C139 P24293 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 300408 T600 C139 P24293 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 300408 T600 C139 P24293 oass.SolrIndexSearcher.<init> Opening Searcher@413c6f94 main
[junit4:junit4] 2> 300409 T599 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@413c6f94 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 300410 T600 C139 P24293 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index.20130530194736027 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index.20130530194736027;done=true>>]
[junit4:junit4] 2> 300410 T600 C139 P24293 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index.20130530194736027
[junit4:junit4] 2> 300410 T600 C139 P24293 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index.20130530194736027
[junit4:junit4] 2> 300411 T600 C139 P24293 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 300411 T600 C139 P24293 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 300411 T600 C139 P24293 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 300411 T600 C139 P24293 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 300418 T600 C139 P24293 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 301051 T529 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 301052 T529 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:24293__collection1",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:24293_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:24293"}
[junit4:junit4] 2> 301064 T548 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 301064 T535 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 301064 T580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 301064 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 301064 T528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 301064 T564 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 301380 T506 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 301382 T506 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
[junit4:junit4] 2> ASYNC NEW_CORE C143 name=collection1 org.apache.solr.core.SolrCore@79641ab1 url=http://127.0.0.1:27471/collection1 node=127.0.0.1:27471_ C143_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:27471_, base_url=http://127.0.0.1:27471, leader=true}
[junit4:junit4] 2> 301391 T518 C143 P27471 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 301394 T518 C143 P27471 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1f002d1f lockFactory=org.apache.lucene.store.NativeFSLockFactory@5e0cb13b),segFN=segments_1,generation=1}
[junit4:junit4] 2> 301395 T518 C143 P27471 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 301396 T518 C143 P27471 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1f002d1f lockFactory=org.apache.lucene.store.NativeFSLockFactory@5e0cb13b),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1f002d1f lockFactory=org.apache.lucene.store.NativeFSLockFactory@5e0cb13b),segFN=segments_2,generation=2}
[junit4:junit4] 2> 301396 T518 C143 P27471 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 301397 T518 C143 P27471 oass.SolrIndexSearcher.<init> Opening Searcher@d903f07 main
[junit4:junit4] 2> 301397 T518 C143 P27471 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 301398 T532 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@d903f07 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 301399 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 8
[junit4:junit4] 2> 301411 T542 C141 P17335 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 301412 T542 C141 P17335 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@651b289f lockFactory=org.apache.lucene.store.NativeFSLockFactory@695eb69a),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@651b289f lockFactory=org.apache.lucene.store.NativeFSLockFactory@695eb69a),segFN=segments_3,generation=3}
[junit4:junit4] 2> 301413 T542 C141 P17335 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 301413 T542 C141 P17335 oass.SolrIndexSearcher.<init> Opening Searcher@5a7169a1 main
[junit4:junit4] 2> 301414 T542 C141 P17335 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 301415 T550 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5a7169a1 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 301415 T542 C141 P17335 oasu.SolrCmdDistributor.distribCommit Distrib commit to:[StdNode: http://127.0.0.1:12138/collection1/, StdNode: http://127.0.0.1:39715/collection1/, StdNode: http://127.0.0.1:24293/collection1/] params:commit_end_point=true&commit=true&softCommit=false&waitSearcher=true&expungeDeletes=false
[junit4:junit4] 2> 301419 T558 C142 P39715 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 301420 T558 C142 P39715 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32ca113e lockFactory=org.apache.lucene.store.NativeFSLockFactory@a90b3f7),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32ca113e lockFactory=org.apache.lucene.store.NativeFSLockFactory@a90b3f7),segFN=segments_3,generation=3}
[junit4:junit4] 2> 301420 T558 C142 P39715 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 301421 T558 C142 P39715 oass.SolrIndexSearcher.<init> Opening Searcher@62d3db5a main
[junit4:junit4] 2> 301421 T558 C142 P39715 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 301422 T566 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@62d3db5a main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 301423 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 4
[junit4:junit4] 2> C140_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:12138_, base_url=http://127.0.0.1:12138}
[junit4:junit4] 2> 301425 T570 C140 P12138 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> C139_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:24293_, base_url=http://127.0.0.1:24293}
[junit4:junit4] 2> 301425 T587 C139 P24293 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 301427 T570 C140 P12138 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32c98179 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3ab0d58),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32c98179 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3ab0d58),segFN=segments_3,generation=3}
[junit4:junit4] 2> 301427 T587 C139 P24293 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@198d7173 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7da22e4a),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(org.apache.lucene.store.RAMDirectory@198d7173 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7da22e4a),segFN=segments_3,generation=3}
[junit4:junit4] 2> 301428 T570 C140 P12138 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 301428 T587 C139 P24293 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 301429 T570 C140 P12138 oass.SolrIndexSearcher.<init> Opening Searcher@71104ed4 main
[junit4:junit4] 2> 301429 T587 C139 P24293 oass.SolrIndexSearcher.<init> Opening Searcher@564d82c2 main
[junit4:junit4] 2> 301430 T570 C140 P12138 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 301430 T587 C139 P24293 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 301430 T582 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@71104ed4 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 301431 T599 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@564d82c2 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 301431 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 6
[junit4:junit4] 2> 301432 T587 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 7
[junit4:junit4] 2> 301433 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 21
[junit4:junit4] 2> 301434 T506 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 301437 T543 C141 P17335 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 301439 T571 C140 P12138 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 301442 T559 C142 P39715 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1
[junit4:junit4] 2> 301444 T588 C139 P24293 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1
[junit4:junit4] 2> 303454 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1436504722323275776)} 0 7
[junit4:junit4] 2> 303462 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1436504722332712960&update.from=http://127.0.0.1:39715/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1436504722332712960)} 0 1
[junit4:junit4] 2> 303466 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1436504722336907264&update.from=http://127.0.0.1:17335/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1436504722336907264)} 0 1
[junit4:junit4] 2> 303467 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=TOLEADER&wt=javabin&version=2} {deleteByQuery=*:* (-1436504722336907264)} 0 6
[junit4:junit4] 2> 303468 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1436504722332712960)} 0 11
[junit4:junit4] 2> 303472 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[0 (1436504722347393024)]} 0 1
[junit4:junit4] 2> 303481 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[0 (1436504722353684480)]} 0 1
[junit4:junit4] 2> 303481 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[0 (1436504722353684480)]} 0 4
[junit4:junit4] 2> 303482 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[0]} 0 7
[junit4:junit4] 2> 303485 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[1 (1436504722362073088)]} 0 0
[junit4:junit4] 2> 303491 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[1 (1436504722365218816)]} 0 0
[junit4:junit4] 2> 303492 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[1 (1436504722365218816)]} 0 4
[junit4:junit4] 2> 303495 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[2 (1436504722372558848)]} 0 0
[junit4:junit4] 2> 303502 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[2 (1436504722375704576)]} 0 1
[junit4:junit4] 2> 303502 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[2 (1436504722375704576)]} 0 4
[junit4:junit4] 2> 303505 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[3 (1436504722383044608)]} 0 0
[junit4:junit4] 2> 303511 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[3 (1436504722386190336)]} 0 0
[junit4:junit4] 2> 303512 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[3 (1436504722386190336)]} 0 4
[junit4:junit4] 2> 303515 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[4 (1436504722393530368)]} 0 0
[junit4:junit4] 2> 303521 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[4 (1436504722396676096)]} 0 0
[junit4:junit4] 2> 303522 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[4 (1436504722396676096)]} 0 4
[junit4:junit4] 2> 303525 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[5 (1436504722404016128)]} 0 0
[junit4:junit4] 2> 303531 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[5 (1436504722407161856)]} 0 0
[junit4:junit4] 2> 303532 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[5 (1436504722407161856)]} 0 4
[junit4:junit4] 2> 303535 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[6 (1436504722414501888)]} 0 0
[junit4:junit4] 2> 303541 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[6 (1436504722417647616)]} 0 0
[junit4:junit4] 2> 303542 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[6 (1436504722417647616)]} 0 4
[junit4:junit4] 2> 303545 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[7 (1436504722424987648)]} 0 0
[junit4:junit4] 2> 303553 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[7 (1436504722430230528)]} 0 0
[junit4:junit4] 2> 303554 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[7 (1436504722430230528)]} 0 4
[junit4:junit4] 2> 303555 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[7]} 0 7
[junit4:junit4] 2> 303559 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[8 (1436504722438619136)]} 0 1
[junit4:junit4] 2> 303567 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[8 (1436504722444910592)]} 0 0
[junit4:junit4] 2> 303568 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[8 (1436504722444910592)]} 0 4
[junit4:junit4] 2> 303569 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[8]} 0 8
[junit4:junit4] 2> 303571 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[9 (1436504722452250624)]} 0 0
[junit4:junit4] 2> 303579 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[9 (1436504722457493504)]} 0 0
[junit4:junit4] 2> 303580 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[9 (1436504722457493504)]} 0 4
[junit4:junit4] 2> 303581 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[9]} 0 7
[junit4:junit4] 2> 303584 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[10 (1436504722465882112)]} 0 0
[junit4:junit4] 2> 303590 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[10 (1436504722469027840)]} 0 0
[junit4:junit4] 2> 303591 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[10 (1436504722469027840)]} 0 4
[junit4:junit4] 2> 303594 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[11 (1436504722476367872)]} 0 0
[junit4:junit4] 2> 303602 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[11 (1436504722481610752)]} 0 0
[junit4:junit4] 2> 303603 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[11 (1436504722481610752)]} 0 4
[junit4:junit4] 2> 303604 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[11]} 0 7
[junit4:junit4] 2> 303607 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[12 (1436504722489999360)]} 0 0
[junit4:junit4] 2> 303616 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[12 (1436504722495242240)]} 0 0
[junit4:junit4] 2> 303616 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[12 (1436504722495242240)]} 0 4
[junit4:junit4] 2> 303617 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[12]} 0 7
[junit4:junit4] 2> 303620 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[13 (1436504722503630848)]} 0 0
[junit4:junit4] 2> 303626 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[13 (1436504722506776576)]} 0 0
[junit4:junit4] 2> 303627 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[13 (1436504722506776576)]} 0 4
[junit4:junit4] 2> 303630 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[14 (1436504722514116608)]} 0 0
[junit4:junit4] 2> 303638 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[14 (1436504722519359488)]} 0 0
[junit4:junit4] 2> 303639 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[14 (1436504722519359488)]} 0 4
[junit4:junit4] 2> 303640 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[14]} 0 7
[junit4:junit4] 2> 303643 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[15 (1436504722527748096)]} 0 0
[junit4:junit4] 2> 303649 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[15 (1436504722530893824)]} 0 0
[junit4:junit4] 2> 303650 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[15 (1436504722530893824)]} 0 4
[junit4:junit4] 2> 303653 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[16 (1436504722538233856)]} 0 0
[junit4:junit4] 2> 303661 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[16 (1436504722543476736)]} 0 0
[junit4:junit4] 2> 303662 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[16 (1436504722543476736)]} 0 4
[junit4:junit4] 2> 303663 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[16]} 0 7
[junit4:junit4] 2> 303666 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[17 (1436504722551865344)]} 0 0
[junit4:junit4] 2> 303674 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[17 (1436504722557108224)]} 0 0
[junit4:junit4] 2> 303675 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[17 (1436504722557108224)]} 0 4
[junit4:junit4] 2> 303676 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[17]} 0 7
[junit4:junit4] 2> 303679 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[18 (1436504722565496832)]} 0 0
[junit4:junit4] 2> 303685 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[18 (1436504722568642560)]} 0 0
[junit4:junit4] 2> 303686 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[18 (1436504722568642560)]} 0 4
[junit4:junit4] 2> 303689 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[19 (1436504722575982592)]} 0 0
[junit4:junit4] 2> 303697 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[19 (1436504722581225472)]} 0 0
[junit4:junit4] 2> 303698 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[19 (1436504722581225472)]} 0 4
[junit4:junit4] 2> 303699 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[19]} 0 7
[junit4:junit4] 2> 303702 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[20 (1436504722589614080)]} 0 0
[junit4:junit4] 2> 303711 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[20 (1436504722594856960)]} 0 0
[junit4:junit4] 2> 303711 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[20 (1436504722594856960)]} 0 4
[junit4:junit4] 2> 303712 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[20]} 0 7
[junit4:junit4] 2> 303715 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[21 (1436504722603245568)]} 0 0
[junit4:junit4] 2> 303723 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[21 (1436504722608488448)]} 0 0
[junit4:junit4] 2> 303724 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[21 (1436504722608488448)]} 0 4
[junit4:junit4] 2> 303725 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[21]} 0 7
[junit4:junit4] 2> 303728 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[22 (1436504722616877056)]} 0 0
[junit4:junit4] 2> 303736 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[22 (1436504722622119936)]} 0 0
[junit4:junit4] 2> 303737 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[22 (1436504722622119936)]} 0 4
[junit4:junit4] 2> 303738 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[22]} 0 7
[junit4:junit4] 2> 303741 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[23 (1436504722630508544)]} 0 0
[junit4:junit4] 2> 303749 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[23 (1436504722635751424)]} 0 0
[junit4:junit4] 2> 303750 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[23 (1436504722635751424)]} 0 4
[junit4:junit4] 2> 303751 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[23]} 0 7
[junit4:junit4] 2> 303755 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[24 (1436504722644140032)]} 0 1
[junit4:junit4] 2> 303761 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[24 (1436504722648334336)]} 0 0
[junit4:junit4] 2> 303762 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[24 (1436504722648334336)]} 0 4
[junit4:junit4] 2> 303766 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[25 (1436504722655674368)]} 0 0
[junit4:junit4] 2> 303774 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[25 (1436504722660917248)]} 0 1
[junit4:junit4] 2> 303774 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[25 (1436504722660917248)]} 0 4
[junit4:junit4] 2> 303775 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[25]} 0 7
[junit4:junit4] 2> 303778 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[26 (1436504722669305856)]} 0 0
[junit4:junit4] 2> 303784 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[26 (1436504722672451584)]} 0 0
[junit4:junit4] 2> 303785 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[26 (1436504722672451584)]} 0 4
[junit4:junit4] 2> 303788 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[27 (1436504722679791616)]} 0 0
[junit4:junit4] 2> 303797 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[27 (1436504722686083072)]} 0 0
[junit4:junit4] 2> 303798 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[27 (1436504722686083072)]} 0 4
[junit4:junit4] 2> 303799 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[27]} 0 8
[junit4:junit4] 2> 303802 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[28 (1436504722694471680)]} 0 0
[junit4:junit4] 2> 303809 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[28 (1436504722697617408)]} 0 1
[junit4:junit4] 2> 303809 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[28 (1436504722697617408)]} 0 4
[junit4:junit4] 2> 303812 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[29 (1436504722704957440)]} 0 0
[junit4:junit4] 2> 303820 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[29 (1436504722710200320)]} 0 0
[junit4:junit4] 2> 303821 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[29 (1436504722710200320)]} 0 4
[junit4:junit4] 2> 303822 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[29]} 0 7
[junit4:junit4] 2> 303825 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[30 (1436504722718588928)]} 0 0
[junit4:junit4] 2> 303832 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[30 (1436504722721734656)]} 0 0
[junit4:junit4] 2> 303832 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[30 (1436504722721734656)]} 0 4
[junit4:junit4] 2> 303835 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[31 (1436504722729074688)]} 0 0
[junit4:junit4] 2> 303843 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[31 (1436504722734317568)]} 0 0
[junit4:junit4] 2> 303844 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[31 (1436504722734317568)]} 0 4
[junit4:junit4] 2> 303845 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[31]} 0 7
[junit4:junit4] 2> 303848 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[32 (1436504722742706176)]} 0 0
[junit4:junit4] 2> 303854 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[32 (1436504722745851904)]} 0 0
[junit4:junit4] 2> 303855 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[32 (1436504722745851904)]} 0 4
[junit4:junit4] 2> 303858 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[33 (1436504722753191936)]} 0 0
[junit4:junit4] 2> 303864 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[33 (1436504722756337664)]} 0 0
[junit4:junit4] 2> 303865 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[33 (1436504722756337664)]} 0 4
[junit4:junit4] 2> 303868 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[34 (1436504722763677696)]} 0 0
[junit4:junit4] 2> 303874 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[34 (1436504722766823424)]} 0 0
[junit4:junit4] 2> 303875 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[34 (1436504722766823424)]} 0 4
[junit4:junit4] 2> 303878 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[35 (1436504722774163456)]} 0 0
[junit4:junit4] 2> 303884 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[35 (1436504722777309184)]} 0 0
[junit4:junit4] 2> 303885 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[35 (1436504722777309184)]} 0 4
[junit4:junit4] 2> 303888 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[36 (1436504722784649216)]} 0 0
[junit4:junit4] 2> 303894 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[36 (1436504722787794944)]} 0 0
[junit4:junit4] 2> 303895 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[36 (1436504722787794944)]} 0 4
[junit4:junit4] 2> 303898 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[37 (1436504722795134976)]} 0 0
[junit4:junit4] 2> 303904 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[37 (1436504722798280704)]} 0 0
[junit4:junit4] 2> 303905 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[37 (1436504722798280704)]} 0 4
[junit4:junit4] 2> 303908 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[38 (1436504722805620736)]} 0 0
[junit4:junit4] 2> 303916 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[38 (1436504722810863616)]} 0 0
[junit4:junit4] 2> 303917 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[38 (1436504722810863616)]} 0 4
[junit4:junit4] 2> 303918 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[38]} 0 7
[junit4:junit4] 2> 303921 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[39 (1436504722819252224)]} 0 0
[junit4:junit4] 2> 303929 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[39 (1436504722824495104)]} 0 0
[junit4:junit4] 2> 303930 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[39 (1436504722824495104)]} 0 4
[junit4:junit4] 2> 303931 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[39]} 0 7
[junit4:junit4] 2> 303934 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[40 (1436504722832883712)]} 0 0
[junit4:junit4] 2> 303940 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[40 (1436504722836029440)]} 0 0
[junit4:junit4] 2> 303941 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[40 (1436504722836029440)]} 0 4
[junit4:junit4] 2> 303944 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[41 (1436504722843369472)]} 0 0
[junit4:junit4] 2> 303952 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[41 (1436504722848612352)]} 0 0
[junit4:junit4] 2> 303953 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[41 (1436504722848612352)]} 0 4
[junit4:junit4] 2> 303954 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[41]} 0 7
[junit4:junit4] 2> 303957 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[42 (1436504722857000960)]} 0 0
[junit4:junit4] 2> 303963 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[42 (1436504722860146688)]} 0 0
[junit4:junit4] 2> 303964 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[42 (1436504722860146688)]} 0 4
[junit4:junit4] 2> 303967 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[43 (1436504722867486720)]} 0 0
[junit4:junit4] 2> 303973 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[43 (1436504722870632448)]} 0 0
[junit4:junit4] 2> 303974 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[43 (1436504722870632448)]} 0 4
[junit4:junit4] 2> 303977 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[44 (1436504722877972480)]} 0 0
[junit4:junit4] 2> 303985 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[44 (1436504722883215360)]} 0 0
[junit4:junit4] 2> 303986 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[44 (1436504722883215360)]} 0 4
[junit4:junit4] 2> 303987 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[44]} 0 7
[junit4:junit4] 2> 303990 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[45 (1436504722891603968)]} 0 0
[junit4:junit4] 2> 303998 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[45 (1436504722896846848)]} 0 0
[junit4:junit4] 2> 303999 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[45 (1436504722896846848)]} 0 4
[junit4:junit4] 2> 304000 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[45]} 0 7
[junit4:junit4] 2> 304003 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[46 (1436504722905235456)]} 0 0
[junit4:junit4] 2> 304011 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[46 (1436504722910478336)]} 0 0
[junit4:junit4] 2> 304012 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[46 (1436504722910478336)]} 0 4
[junit4:junit4] 2> 304013 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[46]} 0 7
[junit4:junit4] 2> 304017 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[47 (1436504722918866944)]} 0 1
[junit4:junit4] 2> 304024 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[47 (1436504722923061248)]} 0 1
[junit4:junit4] 2> 304024 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[47 (1436504722923061248)]} 0 4
[junit4:junit4] 2> 304029 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[48 (1436504722931449856)]} 0 1
[junit4:junit4] 2> 304036 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[48 (1436504722935644160)]} 0 1
[junit4:junit4] 2> 304037 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[48 (1436504722935644160)]} 0 5
[junit4:junit4] 2> 304041 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[49 (1436504722944032768)]} 0 1
[junit4:junit4] 2> 304052 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[49 (1436504722952421376)]} 0 1
[junit4:junit4] 2> 304053 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[49 (1436504722952421376)]} 0 6
[junit4:junit4] 2> 304054 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[49]} 0 10
[junit4:junit4] 2> 304057 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[50 (1436504722961858560)]} 0 0
[junit4:junit4] 2> 304063 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[50 (1436504722965004288)]} 0 0
[junit4:junit4] 2> 304064 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[50 (1436504722965004288)]} 0 4
[junit4:junit4] 2> 304068 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[51 (1436504722972344320)]} 0 1
[junit4:junit4] 2> 304075 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[51 (1436504722977587200)]} 0 0
[junit4:junit4] 2> 304076 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[51 (1436504722977587200)]} 0 4
[junit4:junit4] 2> 304077 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[51]} 0 7
[junit4:junit4] 2> 304080 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[52 (1436504722985975808)]} 0 0
[junit4:junit4] 2> 304088 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[52 (1436504722991218688)]} 0 0
[junit4:junit4] 2> 304089 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[52 (1436504722991218688)]} 0 4
[junit4:junit4] 2> 304090 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[52]} 0 7
[junit4:junit4] 2> 304093 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[53 (1436504722999607296)]} 0 0
[junit4:junit4] 2> 304101 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[53 (1436504723004850176)]} 0 0
[junit4:junit4] 2> 304102 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[53 (1436504723004850176)]} 0 4
[junit4:junit4] 2> 304103 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[53]} 0 7
[junit4:junit4] 2> 304106 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[54 (1436504723013238784)]} 0 0
[junit4:junit4] 2> 304112 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[54 (1436504723016384512)]} 0 0
[junit4:junit4] 2> 304113 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[54 (1436504723016384512)]} 0 4
[junit4:junit4] 2> 304116 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[55 (1436504723023724544)]} 0 0
[junit4:junit4] 2> 304122 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[55 (1436504723026870272)]} 0 0
[junit4:junit4] 2> 304123 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[55 (1436504723026870272)]} 0 4
[junit4:junit4] 2> 304126 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[56 (1436504723034210304)]} 0 0
[junit4:junit4] 2> 304132 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[56 (1436504723037356032)]} 0 0
[junit4:junit4] 2> 304133 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[56 (1436504723037356032)]} 0 4
[junit4:junit4] 2> 304136 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[57 (1436504723044696064)]} 0 0
[junit4:junit4] 2> 304144 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[57 (1436504723049938944)]} 0 0
[junit4:junit4] 2> 304145 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[57 (1436504723049938944)]} 0 4
[junit4:junit4] 2> 304146 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[57]} 0 7
[junit4:junit4] 2> 304149 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[58 (1436504723058327552)]} 0 0
[junit4:junit4] 2> 304157 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[58 (1436504723063570432)]} 0 0
[junit4:junit4] 2> 304158 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[58 (1436504723063570432)]} 0 4
[junit4:junit4] 2> 304159 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[58]} 0 7
[junit4:junit4] 2> 304162 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[59 (1436504723071959040)]} 0 0
[junit4:junit4] 2> 304168 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[59 (1436504723075104768)]} 0 0
[junit4:junit4] 2> 304169 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[59 (1436504723075104768)]} 0 4
[junit4:junit4] 2> 304172 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[60 (1436504723082444800)]} 0 0
[junit4:junit4] 2> 304178 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[60 (1436504723085590528)]} 0 0
[junit4:junit4] 2> 304179 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[60 (1436504723085590528)]} 0 4
[junit4:junit4] 2> 304182 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[61 (1436504723092930560)]} 0 0
[junit4:junit4] 2> 304190 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[61 (1436504723098173440)]} 0 0
[junit4:junit4] 2> 304191 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[61 (1436504723098173440)]} 0 4
[junit4:junit4] 2> 304192 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[61]} 0 7
[junit4:junit4] 2> 304195 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[62 (1436504723106562048)]} 0 0
[junit4:junit4] 2> 304203 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[62 (1436504723111804928)]} 0 0
[junit4:junit4] 2> 304204 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[62 (1436504723111804928)]} 0 4
[junit4:junit4] 2> 304205 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[62]} 0 7
[junit4:junit4] 2> 304208 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[63 (1436504723120193536)]} 0 0
[junit4:junit4] 2> 304214 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[63 (1436504723123339264)]} 0 0
[junit4:junit4] 2> 304215 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[63 (1436504723123339264)]} 0 4
[junit4:junit4] 2> 304218 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[64 (1436504723130679296)]} 0 0
[junit4:junit4] 2> 304224 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[64 (1436504723133825024)]} 0 0
[junit4:junit4] 2> 304225 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[64 (1436504723133825024)]} 0 4
[junit4:junit4] 2> 304228 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[65 (1436504723141165056)]} 0 0
[junit4:junit4] 2> 304236 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[65 (1436504723146407936)]} 0 0
[junit4:junit4] 2> 304237 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[65 (1436504723146407936)]} 0 4
[junit4:junit4] 2> 304238 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[65]} 0 7
[junit4:junit4] 2> 304241 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[66 (1436504723154796544)]} 0 0
[junit4:junit4] 2> 304249 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[66 (1436504723160039424)]} 0 0
[junit4:junit4] 2> 304250 T544 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[66 (1436504723160039424)]} 0 4
[junit4:junit4] 2> 304251 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[66]} 0 7
[junit4:junit4] 2> 304254 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[67 (1436504723168428032)]} 0 0
[junit4:junit4] 2> 304262 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[67 (1436504723173670912)]} 0 0
[junit4:junit4] 2> 304263 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[67 (1436504723173670912)]} 0 4
[junit4:junit4] 2> 304264 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[67]} 0 7
[junit4:junit4] 2> 304267 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[68 (1436504723182059520)]} 0 0
[junit4:junit4] 2> 304273 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[68 (1436504723185205248)]} 0 0
[junit4:junit4] 2> 304274 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[68 (1436504723185205248)]} 0 4
[junit4:junit4] 2> 304277 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[69 (1436504723192545280)]} 0 0
[junit4:junit4] 2> 304283 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[69 (1436504723195691008)]} 0 0
[junit4:junit4] 2> 304284 T560 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[69 (1436504723195691008)]} 0 4
[junit4:junit4] 2> 304287 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[70 (1436504723203031040)]} 0 0
[junit4:junit4] 2> 304293 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[70 (1436504723206176768)]} 0 0
[junit4:junit4] 2> 304294 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[70 (1436504723206176768)]} 0 4
[junit4:junit4] 2> 304297 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[71 (1436504723213516800)]} 0 0
[junit4:junit4] 2> 304305 T589 C139 P24293 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:39715/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[71 (1436504723218759680)]} 0 0
[junit4:junit4] 2> 304306 T558 C142 P39715 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[71 (1436504723218759680)]} 0 4
[junit4:junit4] 2> 304307 T542 C141 P17335 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[71]} 0 7
[junit4:junit4] 2> 304310 T518 C143 P27471 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[72 (1436504723227148288)]} 0 0
[junit4:junit4] 2> 304316 T570 C140 P12138 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:17335/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2
[...truncated too long message...]
.0.0.1:12138_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:12138"}
[junit4:junit4] 2> 569697 T663 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"127.0.0.1:12138__collection1_shard1_0_replica1",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1_0",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":null,
[junit4:junit4] 2> "core":"collection1_shard1_0_replica1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:12138_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:12138"}
[junit4:junit4] 2> 569703 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 569717 T506 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 24293
[junit4:junit4] 2> 569718 T506 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1840278909
[junit4:junit4] 2> 570721 T506 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 570721 T506 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 570724 T506 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@3ac8119
[junit4:junit4] 2> 570726 T506 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=935,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
[junit4:junit4] 2> 570727 T506 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
[junit4:junit4] 2> 570727 T506 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
[junit4:junit4] 2> 570728 T506 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
[junit4:junit4] 2> 570730 T506 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
[junit4:junit4] 2> 570730 T506 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
[junit4:junit4] 2> 570731 T506 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4;done=false>>]
[junit4:junit4] 2> 570731 T506 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4
[junit4:junit4] 2> 570732 T506 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index;done=false>>]
[junit4:junit4] 2> 570732 T506 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534/jetty4/index
[junit4:junit4] 2> 570756 T663 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=89781543406272524-127.0.0.1:24293_-n_0000000004) am no longer a leader.
[junit4:junit4] 2> 570758 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 570758 T597 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 570758 T506 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
[junit4:junit4] 2> 570758 T597 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 570831 T506 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 570833 T506 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:27818 27818
[junit4:junit4] 2> 570966 T580 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 570981 T580 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 570982 T580 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 570983 T506 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 27471
[junit4:junit4] 2> !!!! WARNING: best effort to remove /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369957632534 FAILED !!!!!
[junit4:junit4] 2> 570984 T506 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 570985 T506 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:27818 27818
[junit4:junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4:junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeyShardSplitTest -Dtests.method=testDistribSearch -Dtests.seed=584A30134BDBB210 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_PY -Dtests.timezone=America/Nipigon -Dtests.file.encoding=US-ASCII
[junit4:junit4] ERROR 294s J1 | ChaosMonkeyShardSplitTest.testDistribSearch <<<
[junit4:junit4] > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:17335 returned non ok status:500, message:Server Error
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([584A30134BDBB210:D9ACBE0B3C84D22C]:0)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4:junit4] > at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:208)
[junit4:junit4] > at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:136)
[junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:679)
[junit4:junit4] 2> 571158 T506 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4] 2> 294262 T505 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
[junit4:junit4] 2> NOTE: test params are: codec=Lucene42: {id=Pulsing41(freqCutoff=19 minBlockSize=71 maxBlockSize=189), _version_=PostingsFormat(name=Memory doPackFST= true)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=es_PY, timezone=America/Nipigon
[junit4:junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Sun Microsystems Inc. 1.6.0_32 (64-bit)/cpus=16,threads=2,free=234792904,total=448069632
[junit4:junit4] 2> NOTE: All tests run in this JVM: [TestSolrXml, TestSystemIdResolver, SuggesterWFSTTest, TestCoreDiscovery, TestAtomicUpdateErrorCases, TestFastOutputStream, TestJmxIntegration, TestComponentsName, TestPostingsSolrHighlighter, CurrencyFieldXmlFileTest, TestFieldTypeResource, TestAnalyzedSuggestions, SortByFunctionTest, TestArbitraryIndexDir, MultiTermTest, ClusterStateTest, TestPartialUpdateDeduplication, TestAddFieldRealTimeGet, IndexBasedSpellCheckerTest, TestBinaryField, PingRequestHandlerTest, OpenExchangeRatesOrgProviderTest, TestDynamicFieldCollectionResource, TestFiltering, TestCopyFieldCollectionResource, PathHierarchyTokenizerFactoryTest, SpellCheckCollatorTest, BadIndexSchemaTest, NumericFieldsTest, RequiredFieldsTest, TestSchemaSimilarityResource, RequestHandlersTest, TestMaxScoreQueryParser, HighlighterTest, FieldAnalysisRequestHandlerTest, TestLMJelinekMercerSimilarityFactory, SignatureUpdateProcessorFactoryTest, TestPropInject, SolrTestCaseJ4Test, TestIndexingPerformance, CurrencyFieldOpenExchangeTest, TestSuggestSpellingConverter, BasicDistributedZk2Test, TestSolrQueryParserResource, ChaosMonkeyShardSplitTest]
[junit4:junit4] Completed on J1 in 294.54s, 1 test, 1 error <<< FAILURES!
[...truncated 681 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:393: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:366: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:181: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:437: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1243: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:887: There were test failures: 296 suites, 1235 tests, 1 error, 11 ignored (5 assumptions)
Total time: 100 minutes 48 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
[JENKINS] Lucene-Solr-NightlyTests-4.x - Build # 271 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-4.x/271/
4 tests failed.
REGRESSION: org.apache.solr.cloud.ChaosMonkeyShardSplitTest.testDistribSearch
Error Message:
Server at http://127.0.0.1:54158 returned non ok status:500, message:Server Error
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:54158 returned non ok status:500, message:Server Error
at __randomizedtesting.SeedInfo.seed([C92FB91E9C81331:8D7475899E97730D]:0)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:208)
at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:136)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:679)
REGRESSION: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch
Error Message:
There are still nodes recoverying - waited for 230 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 230 seconds
at __randomizedtesting.SeedInfo.seed([C92FB91E9C81331:8D7475899E97730D]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:173)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:131)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:126)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:512)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:146)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:679)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 1) Thread[id=2887, name=recoveryCmdExecutor-1223-thread-1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:327) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:193) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:180) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:384) at java.net.Socket.connect(Socket.java:546) at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127) at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180) at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294) at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180) at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:298) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:679)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:
1) Thread[id=2887, name=recoveryCmdExecutor-1223-thread-1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:327)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:193)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:180)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:384)
at java.net.Socket.connect(Socket.java:546)
at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:298)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
at __randomizedtesting.SeedInfo.seed([C92FB91E9C81331]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=2887, name=recoveryCmdExecutor-1223-thread-1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest] at java.net.PlainSocketImpl.socketConnect(Native Method) at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:327) at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:193) at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:180) at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:384) at java.net.Socket.connect(Socket.java:546) at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127) at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180) at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294) at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645) at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805) at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365) at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180) at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:298) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:679)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=2887, name=recoveryCmdExecutor-1223-thread-1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:327)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:193)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:180)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:384)
at java.net.Socket.connect(Socket.java:546)
at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:298)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
at __randomizedtesting.SeedInfo.seed([C92FB91E9C81331]:0)
Build Log:
[...truncated 8891 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeyShardSplitTest
[junit4:junit4] 2> 299504 T491 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
[junit4:junit4] 2> 299511 T491 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./solrtest-ChaosMonkeyShardSplitTest-1369857938865
[junit4:junit4] 2> 299513 T491 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 299514 T492 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 299615 T491 oasc.ZkTestServer.run start zk server on port:54151
[junit4:junit4] 2> 299616 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 299648 T498 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4054bbf7 name:ZooKeeperConnection Watcher:127.0.0.1:54151 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 299648 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 299648 T491 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 299667 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 299670 T500 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50c96bbe name:ZooKeeperConnection Watcher:127.0.0.1:54151/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 299670 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 299671 T491 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 299676 T491 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 299684 T491 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 299688 T491 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 299693 T491 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 299694 T491 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 299708 T491 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 299709 T491 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 299814 T491 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 299815 T491 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 299820 T491 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 299821 T491 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 299825 T491 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 299826 T491 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 299831 T491 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 299832 T491 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 299836 T491 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 299837 T491 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 299841 T491 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 299842 T491 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 299847 T491 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 299847 T491 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 300108 T491 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 300112 T491 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54154
[junit4:junit4] 2> 300113 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 300113 T491 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 300114 T491 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369857939206
[junit4:junit4] 2> 300114 T491 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369857939206/solr.xml
[junit4:junit4] 2> 300115 T491 oasc.CoreContainer.<init> New CoreContainer 1851139784
[junit4:junit4] 2> 300116 T491 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369857939206/'
[junit4:junit4] 2> 300116 T491 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369857939206/'
[junit4:junit4] 2> 300232 T491 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 300233 T491 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 300234 T491 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 300234 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 300235 T491 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 300235 T491 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 300236 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 300236 T491 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 300237 T491 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 300237 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 300257 T491 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 300258 T491 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54151/solr
[junit4:junit4] 2> 300258 T491 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 300259 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 300262 T511 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4638d3b7 name:ZooKeeperConnection Watcher:127.0.0.1:54151 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 300263 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 300266 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 300280 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 300283 T513 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ae9f3bf name:ZooKeeperConnection Watcher:127.0.0.1:54151/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 300283 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 300286 T491 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 300297 T491 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 300302 T491 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 300305 T491 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54154_
[junit4:junit4] 2> 300308 T491 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54154_
[junit4:junit4] 2> 300317 T491 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 300333 T491 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 300337 T491 oasc.Overseer.start Overseer (id=89775009881980931-127.0.0.1:54154_-n_0000000000) starting
[junit4:junit4] 2> 300348 T491 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 300360 T515 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 300361 T491 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 300365 T491 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 300368 T491 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 300372 T514 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 300376 T516 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369857939206/collection1
[junit4:junit4] 2> 300376 T516 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 300377 T516 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 300377 T516 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 300379 T516 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369857939206/collection1/'
[junit4:junit4] 2> 300381 T516 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369857939206/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 300382 T516 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369857939206/collection1/lib/README' to classloader
[junit4:junit4] 2> 300436 T516 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 300503 T516 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 300605 T516 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 300617 T516 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 301176 T516 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 301180 T516 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 301182 T516 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 301189 T516 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 301234 T516 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 301234 T516 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1369857939206/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/control/data/
[junit4:junit4] 2> 301234 T516 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2031688f
[junit4:junit4] 2> 301235 T516 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 301236 T516 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/control/data
[junit4:junit4] 2> 301236 T516 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/control/data/index/
[junit4:junit4] 2> 301236 T516 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 301237 T516 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/control/data/index
[junit4:junit4] 2> 301242 T516 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1aeed8ce lockFactory=org.apache.lucene.store.NativeFSLockFactory@300cb3d5; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> 301242 T516 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 301245 T516 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 301245 T516 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 301246 T516 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 301246 T516 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 301247 T516 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 301247 T516 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 301248 T516 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 301249 T516 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 301249 T516 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 301259 T516 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 301266 T516 oass.SolrIndexSearcher.<init> Opening Searcher@59e5c799 main
[junit4:junit4] 2> 301267 T516 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/control/data/tlog
[junit4:junit4] 2> 301268 T516 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 301268 T516 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 301271 T517 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@59e5c799 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 301272 T516 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 301273 T516 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 301878 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 301879 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54154_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54154"}
[junit4:junit4] 2> 301879 T514 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 301879 T514 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 301900 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 302276 T516 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 302276 T516 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54154 collection:control_collection shard:shard1
[junit4:junit4] 2> 302277 T516 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 302301 T516 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 302313 T516 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 302313 T516 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 302314 T516 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54154/collection1/
[junit4:junit4] 2> 302314 T516 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 302314 T516 oasc.SyncStrategy.syncToMe http://127.0.0.1:54154/collection1/ has no replicas
[junit4:junit4] 2> 302315 T516 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54154/collection1/
[junit4:junit4] 2> 302315 T516 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 303408 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 303430 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 303455 T516 oasc.ZkController.register We are http://127.0.0.1:54154/collection1/ and leader is http://127.0.0.1:54154/collection1/
[junit4:junit4] 2> 303455 T516 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54154
[junit4:junit4] 2> 303456 T516 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 303456 T516 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 303456 T516 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 303496 T516 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 303498 T491 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 303499 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 303499 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 303516 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 303518 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 303521 T520 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72463e02 name:ZooKeeperConnection Watcher:127.0.0.1:54151/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 303521 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 303524 T491 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 303527 T491 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 303793 T491 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 303796 T491 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:54158
[junit4:junit4] 2> 303797 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 303797 T491 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 303798 T491 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369857942881
[junit4:junit4] 2> 303799 T491 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369857942881/solr.xml
[junit4:junit4] 2> 303799 T491 oasc.CoreContainer.<init> New CoreContainer 932427961
[junit4:junit4] 2> 303800 T491 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369857942881/'
[junit4:junit4] 2> 303800 T491 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369857942881/'
[junit4:junit4] 2> 303913 T491 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 303914 T491 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 303914 T491 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 303915 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 303915 T491 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 303916 T491 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 303916 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 303917 T491 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 303917 T491 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 303918 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 303935 T491 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 303935 T491 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54151/solr
[junit4:junit4] 2> 303936 T491 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 303937 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 303944 T531 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4be04bd0 name:ZooKeeperConnection Watcher:127.0.0.1:54151 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 303945 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 303948 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 303960 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 303962 T533 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b3d6796 name:ZooKeeperConnection Watcher:127.0.0.1:54151/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 303962 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 303975 T491 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 304939 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 304941 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54154_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54154"}
[junit4:junit4] 2> 304959 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 304959 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 304959 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 304980 T491 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54158_
[junit4:junit4] 2> 304983 T491 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54158_
[junit4:junit4] 2> 304986 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 304987 T520 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 304987 T533 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 304987 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 304987 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 304989 T513 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 304996 T534 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369857942881/collection1
[junit4:junit4] 2> 304996 T534 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 304997 T534 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 304997 T534 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 304999 T534 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369857942881/collection1/'
[junit4:junit4] 2> 305001 T534 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369857942881/collection1/lib/README' to classloader
[junit4:junit4] 2> 305001 T534 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369857942881/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 305053 T534 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 305112 T534 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 305214 T534 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 305225 T534 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 305770 T534 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 305774 T534 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 305777 T534 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 305783 T534 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 305825 T534 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 305826 T534 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1369857942881/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty1/
[junit4:junit4] 2> 305826 T534 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2031688f
[junit4:junit4] 2> 305827 T534 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 305828 T534 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty1
[junit4:junit4] 2> 305828 T534 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty1/index/
[junit4:junit4] 2> 305828 T534 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 305829 T534 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty1/index
[junit4:junit4] 2> 305833 T534 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@584249b2 lockFactory=org.apache.lucene.store.NativeFSLockFactory@466f2973; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> 305833 T534 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 305836 T534 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 305836 T534 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 305837 T534 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 305838 T534 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 305839 T534 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 305839 T534 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 305840 T534 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 305841 T534 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 305841 T534 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 305851 T534 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 305858 T534 oass.SolrIndexSearcher.<init> Opening Searcher@2b4eec1 main
[junit4:junit4] 2> 305859 T534 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty1/tlog
[junit4:junit4] 2> 305860 T534 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 305860 T534 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 305866 T535 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2b4eec1 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 305868 T534 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 305868 T534 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 306466 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 306467 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54158_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54158"}
[junit4:junit4] 2> 306468 T514 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4:junit4] 2> 306468 T514 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 306482 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 306482 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 306482 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 306871 T534 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 306871 T534 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54158 collection:collection1 shard:shard1
[junit4:junit4] 2> 306872 T534 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 306918 T534 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 306930 T534 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 306930 T534 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 306930 T534 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54158/collection1/
[junit4:junit4] 2> 306931 T534 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 306931 T534 oasc.SyncStrategy.syncToMe http://127.0.0.1:54158/collection1/ has no replicas
[junit4:junit4] 2> 306931 T534 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54158/collection1/
[junit4:junit4] 2> 306931 T534 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 307990 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 308011 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 308011 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 308011 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 308018 T534 oasc.ZkController.register We are http://127.0.0.1:54158/collection1/ and leader is http://127.0.0.1:54158/collection1/
[junit4:junit4] 2> 308018 T534 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54158
[junit4:junit4] 2> 308018 T534 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 308018 T534 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 308019 T534 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 308021 T534 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 308023 T491 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 308023 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 308024 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 308294 T491 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 308297 T491 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:40567
[junit4:junit4] 2> 308298 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 308298 T491 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 308299 T491 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369857947395
[junit4:junit4] 2> 308299 T491 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369857947395/solr.xml
[junit4:junit4] 2> 308300 T491 oasc.CoreContainer.<init> New CoreContainer 134907191
[junit4:junit4] 2> 308300 T491 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369857947395/'
[junit4:junit4] 2> 308301 T491 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369857947395/'
[junit4:junit4] 2> 308415 T491 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 308416 T491 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 308417 T491 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 308417 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 308418 T491 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 308418 T491 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 308419 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 308419 T491 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 308420 T491 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 308420 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 308438 T491 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 308438 T491 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54151/solr
[junit4:junit4] 2> 308439 T491 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 308440 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 308443 T547 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bb5a5dd name:ZooKeeperConnection Watcher:127.0.0.1:54151 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 308444 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 308454 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 308467 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 308508 T549 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49743a75 name:ZooKeeperConnection Watcher:127.0.0.1:54151/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 308509 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 308535 T491 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 309521 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 309522 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:54158_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:54158"}
[junit4:junit4] 2> 309540 T491 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40567_
[junit4:junit4] 2> 309552 T491 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40567_
[junit4:junit4] 2> 309553 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 309553 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 309553 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 309553 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 309563 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 309564 T533 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 309564 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 309564 T520 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 309565 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 309565 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 309567 T549 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 309568 T513 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 309573 T550 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369857947395/collection1
[junit4:junit4] 2> 309573 T550 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 309574 T550 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 309574 T550 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 309576 T550 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369857947395/collection1/'
[junit4:junit4] 2> 309578 T550 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369857947395/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 309578 T550 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369857947395/collection1/lib/README' to classloader
[junit4:junit4] 2> 309630 T550 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 309688 T550 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 309790 T550 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 309802 T550 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 310352 T550 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 310356 T550 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 310359 T550 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 310365 T550 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 310402 T550 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 310403 T550 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1369857947395/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty2/
[junit4:junit4] 2> 310403 T550 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2031688f
[junit4:junit4] 2> 310403 T550 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 310404 T550 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty2
[junit4:junit4] 2> 310404 T550 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty2/index/
[junit4:junit4] 2> 310405 T550 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 310405 T550 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty2/index
[junit4:junit4] 2> 310408 T550 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@2c26618f lockFactory=org.apache.lucene.store.NativeFSLockFactory@1b1f4d6f; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> 310408 T550 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 310410 T550 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 310410 T550 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 310411 T550 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 310411 T550 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 310412 T550 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 310412 T550 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 310412 T550 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 310413 T550 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 310413 T550 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 310419 T550 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 310425 T550 oass.SolrIndexSearcher.<init> Opening Searcher@2180a743 main
[junit4:junit4] 2> 310426 T550 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty2/tlog
[junit4:junit4] 2> 310426 T550 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 310427 T550 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 310430 T551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2180a743 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 310432 T550 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 310432 T550 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 311066 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 311067 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:40567_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:40567"}
[junit4:junit4] 2> 311067 T514 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 311067 T514 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 311093 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 311093 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 311093 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 311093 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 311460 T550 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 311460 T550 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:40567 collection:collection1 shard:shard2
[junit4:junit4] 2> 311461 T550 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 311478 T550 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 311489 T550 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 311489 T550 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 311489 T550 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:40567/collection1/
[junit4:junit4] 2> 311489 T550 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 311490 T550 oasc.SyncStrategy.syncToMe http://127.0.0.1:40567/collection1/ has no replicas
[junit4:junit4] 2> 311490 T550 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:40567/collection1/
[junit4:junit4] 2> 311490 T550 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 312599 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 312626 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 312626 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 312626 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 312626 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 312679 T550 oasc.ZkController.register We are http://127.0.0.1:40567/collection1/ and leader is http://127.0.0.1:40567/collection1/
[junit4:junit4] 2> 312679 T550 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:40567
[junit4:junit4] 2> 312679 T550 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 312679 T550 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 312680 T550 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 312682 T550 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 312684 T491 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 312685 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 312685 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 312959 T491 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 312963 T491 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60281
[junit4:junit4] 2> 312963 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 312964 T491 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 312965 T491 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369857952056
[junit4:junit4] 2> 312965 T491 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369857952056/solr.xml
[junit4:junit4] 2> 312966 T491 oasc.CoreContainer.<init> New CoreContainer 585817277
[junit4:junit4] 2> 312966 T491 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369857952056/'
[junit4:junit4] 2> 312967 T491 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369857952056/'
[junit4:junit4] 2> 313077 T491 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 313078 T491 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 313078 T491 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 313079 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 313079 T491 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 313080 T491 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 313080 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 313081 T491 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 313081 T491 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 313082 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 313100 T491 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 313101 T491 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54151/solr
[junit4:junit4] 2> 313101 T491 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 313103 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 313130 T563 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6fb26c41 name:ZooKeeperConnection Watcher:127.0.0.1:54151 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 313131 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 313134 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 313147 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 313150 T565 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@51f739d3 name:ZooKeeperConnection Watcher:127.0.0.1:54151/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 313150 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 313157 T491 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 314136 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 314137 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:40567_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:40567"}
[junit4:junit4] 2> 314156 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 314156 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 314156 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 314156 T565 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 314156 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 314162 T491 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60281_
[junit4:junit4] 2> 314164 T491 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60281_
[junit4:junit4] 2> 314167 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 314167 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 314168 T520 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 314168 T533 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 314168 T565 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 314168 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 314169 T565 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 314168 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 314170 T513 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 314170 T549 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 314177 T566 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369857952056/collection1
[junit4:junit4] 2> 314178 T566 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 314179 T566 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 314179 T566 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 314181 T566 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369857952056/collection1/'
[junit4:junit4] 2> 314183 T566 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369857952056/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 314183 T566 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369857952056/collection1/lib/README' to classloader
[junit4:junit4] 2> 314244 T566 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 314308 T566 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 314410 T566 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 314419 T566 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 314874 T566 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 314877 T566 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 314879 T566 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 314883 T566 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 314918 T566 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 314918 T566 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1369857952056/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/
[junit4:junit4] 2> 314918 T566 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2031688f
[junit4:junit4] 2> 314919 T566 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 314919 T566 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3
[junit4:junit4] 2> 314920 T566 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/index/
[junit4:junit4] 2> 314920 T566 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 314920 T566 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/index
[junit4:junit4] 2> 314923 T566 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@140962cf lockFactory=org.apache.lucene.store.NativeFSLockFactory@52b1eaed; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> 314924 T566 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 314926 T566 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 314926 T566 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 314926 T566 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 314927 T566 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 314927 T566 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 314928 T566 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 314928 T566 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 314928 T566 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 314929 T566 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 314936 T566 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 314941 T566 oass.SolrIndexSearcher.<init> Opening Searcher@6a34e9d0 main
[junit4:junit4] 2> 314941 T566 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/tlog
[junit4:junit4] 2> 314942 T566 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 314943 T566 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 314947 T567 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6a34e9d0 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 314948 T566 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 314949 T566 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 315663 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 315664 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:60281_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:60281"}
[junit4:junit4] 2> 315664 T514 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 315665 T514 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 315678 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 315678 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 315678 T565 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 315678 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 315678 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 315952 T566 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 315952 T566 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:60281 collection:collection1 shard:shard1
[junit4:junit4] 2> 315957 T566 oasc.ZkController.register We are http://127.0.0.1:60281/collection1/ and leader is http://127.0.0.1:54158/collection1/
[junit4:junit4] 2> 315957 T566 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60281
[junit4:junit4] 2> 315957 T566 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 315957 T566 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C571 name=collection1 org.apache.solr.core.SolrCore@3ef99bfd url=http://127.0.0.1:60281/collection1 node=127.0.0.1:60281_ C571_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:60281_, base_url=http://127.0.0.1:60281}
[junit4:junit4] 2> 315963 T568 C571 P60281 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 315964 T568 C571 P60281 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 315964 T566 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 315964 T568 C571 P60281 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 315964 T568 C571 P60281 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 315965 T491 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 315966 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 315966 T568 C571 P60281 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 315967 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 316346 T491 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 316351 T491 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38017
[junit4:junit4] 2> 316352 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 316352 T491 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 316352 T491 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369857955337
[junit4:junit4] 2> 316353 T491 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369857955337/solr.xml
[junit4:junit4] 2> 316353 T491 oasc.CoreContainer.<init> New CoreContainer 1450341966
[junit4:junit4] 2> 316354 T491 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369857955337/'
[junit4:junit4] 2> 316354 T491 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369857955337/'
[junit4:junit4] 2> 316962 T491 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 316963 T491 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 316963 T491 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 316964 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 316964 T491 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 316964 T491 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 316965 T491 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 316965 T491 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 316965 T491 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 316966 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 316980 T491 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 316980 T491 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54151/solr
[junit4:junit4] 2> 316981 T491 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 316982 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 316985 T580 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@613a01ca name:ZooKeeperConnection Watcher:127.0.0.1:54151 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 316986 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 316989 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 316999 T491 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 317001 T582 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6151e258 name:ZooKeeperConnection Watcher:127.0.0.1:54151/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 317003 T491 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 317008 T491 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 317156 T523 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 317185 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 317186 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:60281_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:60281"}
[junit4:junit4] 2> 317202 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 317202 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 317202 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 317202 T565 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 317202 T582 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 317202 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 318014 T491 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38017_
[junit4:junit4] 2> 318016 T491 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38017_
[junit4:junit4] 2> 318020 T582 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 318020 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 318020 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 318021 T565 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 318020 T533 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 318020 T520 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 318023 T582 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 318023 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 318024 T549 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 318022 T565 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 318025 T513 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 318023 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 318035 T583 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369857955337/collection1
[junit4:junit4] 2> 318035 T583 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 318036 T583 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 318037 T583 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 318039 T583 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369857955337/collection1/'
[junit4:junit4] 2> 318041 T583 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369857955337/collection1/lib/README' to classloader
[junit4:junit4] 2> 318042 T583 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369857955337/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 318135 T583 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 318158 T523 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 318159 T523 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=4&state=recovering&nodeName=127.0.0.1:60281_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1008
[junit4:junit4] 2> 318307 T583 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 318409 T583 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 318418 T583 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 319067 T583 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 319072 T583 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 319075 T583 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 319083 T583 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 319127 T583 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 319127 T583 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1369857955337/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/
[junit4:junit4] 2> 319128 T583 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2031688f
[junit4:junit4] 2> 319129 T583 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 319130 T583 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4
[junit4:junit4] 2> 319130 T583 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/index/
[junit4:junit4] 2> 319131 T583 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 319133 T583 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/index
[junit4:junit4] 2> 319138 T583 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@4651534f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6b6f4b51; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> 319138 T583 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 319142 T583 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 319142 T583 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 319143 T583 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 319144 T583 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 319145 T583 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 319146 T583 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 319146 T583 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 319147 T583 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 319148 T583 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 319158 T583 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 319165 T583 oass.SolrIndexSearcher.<init> Opening Searcher@334b7f7f main
[junit4:junit4] 2> 319166 T583 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/tlog
[junit4:junit4] 2> 319167 T583 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 319168 T583 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 319173 T584 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@334b7f7f main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 319175 T583 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 319176 T583 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 320212 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 320213 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:38017_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:38017"}
[junit4:junit4] 2> 320213 T514 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 320213 T514 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 320227 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 320227 T565 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 320227 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 320227 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 320227 T582 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 320227 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> ASYNC NEW_CORE C572 name=collection1 org.apache.solr.core.SolrCore@3ef99bfd url=http://127.0.0.1:60281/collection1 node=127.0.0.1:60281_ C572_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:60281_, base_url=http://127.0.0.1:60281}
[junit4:junit4] 2> 320271 T568 C572 P60281 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:54158/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 320282 T568 C572 P60281 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&connTimeout=30000&socketTimeout=30000&retry=false
[junit4:junit4] 2> 320308 T568 C572 P60281 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:60281 START replicas=[http://127.0.0.1:54158/collection1/] nUpdates=100
[junit4:junit4] 2> 320321 T568 C572 P60281 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 320322 T568 C572 P60281 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 320322 T568 C572 P60281 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 320322 T568 C572 P60281 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 320323 T568 C572 P60281 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 320323 T568 C572 P60281 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:54158/collection1/. core=collection1
[junit4:junit4] 2> 320323 T568 C572 P60281 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C573 name=collection1 org.apache.solr.core.SolrCore@69b82b30 url=http://127.0.0.1:54158/collection1 node=127.0.0.1:54158_ C573_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:54158_, base_url=http://127.0.0.1:54158, leader=true}
[junit4:junit4] 2> 320388 T525 C573 P54158 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=8
[junit4:junit4] 2> 320409 T524 C573 P54158 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 320412 T524 C573 P54158 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@584249b2 lockFactory=org.apache.lucene.store.NativeFSLockFactory@466f2973; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> 320412 T524 C573 P54158 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 320413 T524 C573 P54158 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@584249b2 lockFactory=org.apache.lucene.store.NativeFSLockFactory@466f2973; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@584249b2 lockFactory=org.apache.lucene.store.NativeFSLockFactory@466f2973; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_2,generation=2}
[junit4:junit4] 2> 320413 T524 C573 P54158 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 320414 T524 C573 P54158 oass.SolrIndexSearcher.<init> Opening Searcher@3423ce52 realtime
[junit4:junit4] 2> 320415 T524 C573 P54158 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 320415 T524 C573 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 34
[junit4:junit4] 2> 320490 T568 C572 P60281 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 320491 T568 C572 P60281 oascsi.HttpClientUtil.createClient Creating new http client, config:connTimeout=5000&socketTimeout=20000&allowCompression=false&maxConnections=10000&maxConnectionsPerHost=10000
[junit4:junit4] 2> 320520 T568 C572 P60281 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 320591 T526 C573 P54158 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 320592 T526 C573 P54158 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=69
[junit4:junit4] 2> 320593 T568 C572 P60281 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 320593 T568 C572 P60281 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 320594 T568 C572 P60281 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 320597 T526 C573 P54158 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 320598 T568 C572 P60281 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 320600 T568 C572 P60281 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/index.20130529230559952
[junit4:junit4] 2> 320600 T568 C572 P60281 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@64708d5a lockFactory=org.apache.lucene.store.NativeFSLockFactory@599ad3d9) fullCopy=false
[junit4:junit4] 2> 320615 T526 C573 P54158 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=6
[junit4:junit4] 2> 320632 T568 C572 P60281 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 320633 T568 C572 P60281 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 320633 T568 C572 P60281 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 320636 T568 C572 P60281 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@140962cf lockFactory=org.apache.lucene.store.NativeFSLockFactory@52b1eaed; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@140962cf lockFactory=org.apache.lucene.store.NativeFSLockFactory@52b1eaed; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_2,generation=2}
[junit4:junit4] 2> 320636 T568 C572 P60281 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 320636 T568 C572 P60281 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 320637 T568 C572 P60281 oass.SolrIndexSearcher.<init> Opening Searcher@64bb2d9e main
[junit4:junit4] 2> 320638 T567 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@64bb2d9e main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 320639 T568 C572 P60281 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/index.20130529230559952 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/index.20130529230559952;done=true>>]
[junit4:junit4] 2> 320639 T568 C572 P60281 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/index.20130529230559952
[junit4:junit4] 2> 320639 T568 C572 P60281 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty3/index.20130529230559952
[junit4:junit4] 2> 320640 T568 C572 P60281 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 320640 T568 C572 P60281 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 320640 T568 C572 P60281 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 320640 T568 C572 P60281 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 320642 T568 C572 P60281 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 321180 T583 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 321180 T583 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38017 collection:collection1 shard:shard2
[junit4:junit4] 2> 321186 T583 oasc.ZkController.register We are http://127.0.0.1:38017/collection1/ and leader is http://127.0.0.1:40567/collection1/
[junit4:junit4] 2> 321186 T583 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38017
[junit4:junit4] 2> 321186 T583 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 321186 T583 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C574 name=collection1 org.apache.solr.core.SolrCore@17002d90 url=http://127.0.0.1:38017/collection1 node=127.0.0.1:38017_ C574_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:38017_, base_url=http://127.0.0.1:38017}
[junit4:junit4] 2> 321187 T587 C574 P38017 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 321188 T587 C574 P38017 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 321188 T583 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 321188 T587 C574 P38017 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 321189 T587 C574 P38017 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 321190 T491 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build/solr-core/test/J1
[junit4:junit4] 2> 321190 T491 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 321191 T491 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 321196 T587 C574 P38017 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 321207 T491 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 321209 T491 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 15 for each attempt
[junit4:junit4] 2> 321209 T491 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4:junit4] 2> 321210 T491 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 321215 T539 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 321733 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 321735 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:60281_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:60281"}
[junit4:junit4] 2> 321740 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:38017_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:38017"}
[junit4:junit4] 2> 321752 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 321752 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 321752 T582 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 321752 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 321752 T565 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 321752 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 322213 T491 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 322217 T539 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 322217 T539 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:38017_&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1002
[junit4:junit4] 2> 323217 T491 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> C574_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:38017_, base_url=http://127.0.0.1:38017}
[junit4:junit4] 2> 324219 T587 C574 P38017 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:40567/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 324219 T587 C574 P38017 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:38017 START replicas=[http://127.0.0.1:40567/collection1/] nUpdates=100
[junit4:junit4] 2> 324220 T491 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 324220 T587 C574 P38017 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 324221 T587 C574 P38017 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 324223 T587 C574 P38017 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 324224 T587 C574 P38017 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 324224 T587 C574 P38017 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 324224 T587 C574 P38017 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:40567/collection1/. core=collection1
[junit4:junit4] 2> 324224 T587 C574 P38017 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C575 name=collection1 org.apache.solr.core.SolrCore@765bf112 url=http://127.0.0.1:40567/collection1 node=127.0.0.1:40567_ C575_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:40567_, base_url=http://127.0.0.1:40567, leader=true}
[junit4:junit4] 2> 324235 T540 C575 P40567 oasc.SolrCore.execute [collection1] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1
[junit4:junit4] 2> 324238 T541 C575 P40567 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 324240 T541 C575 P40567 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@2c26618f lockFactory=org.apache.lucene.store.NativeFSLockFactory@1b1f4d6f; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> 324241 T541 C575 P40567 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 324241 T541 C575 P40567 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@2c26618f lockFactory=org.apache.lucene.store.NativeFSLockFactory@1b1f4d6f; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@2c26618f lockFactory=org.apache.lucene.store.NativeFSLockFactory@1b1f4d6f; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_2,generation=2}
[junit4:junit4] 2> 324242 T541 C575 P40567 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 324242 T541 C575 P40567 oass.SolrIndexSearcher.<init> Opening Searcher@2f27d403 realtime
[junit4:junit4] 2> 324243 T541 C575 P40567 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 324243 T541 C575 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 6
[junit4:junit4] 2> 324244 T587 C574 P38017 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 324245 T587 C574 P38017 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 324247 T542 C575 P40567 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 324248 T542 C575 P40567 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=1
[junit4:junit4] 2> 324248 T587 C574 P38017 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 324249 T587 C574 P38017 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 324249 T587 C574 P38017 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 324253 T542 C575 P40567 oasc.SolrCore.execute [collection1] webapp= path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 324254 T587 C574 P38017 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 324255 T587 C574 P38017 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/index.20130529230603607
[junit4:junit4] 2> 324255 T587 C574 P38017 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@326d14e2 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5c26b747) fullCopy=false
[junit4:junit4] 2> 324259 T542 C575 P40567 oasc.SolrCore.execute [collection1] webapp= path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=1
[junit4:junit4] 2> 324260 T587 C574 P38017 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 324261 T587 C574 P38017 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 324261 T587 C574 P38017 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 324263 T587 C574 P38017 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@4651534f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6b6f4b51; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@4651534f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6b6f4b51; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_2,generation=2}
[junit4:junit4] 2> 324264 T587 C574 P38017 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 324264 T587 C574 P38017 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 324265 T587 C574 P38017 oass.SolrIndexSearcher.<init> Opening Searcher@13204f73 main
[junit4:junit4] 2> 324266 T584 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@13204f73 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 324266 T587 C574 P38017 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/index.20130529230603607 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/index.20130529230603607;done=true>>]
[junit4:junit4] 2> 324267 T587 C574 P38017 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/index.20130529230603607
[junit4:junit4] 2> 324267 T587 C574 P38017 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1369857938865/jetty4/index.20130529230603607
[junit4:junit4] 2> 324267 T587 C574 P38017 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 324267 T587 C574 P38017 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 324268 T587 C574 P38017 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 324268 T587 C574 P38017 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 324280 T587 C574 P38017 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 324770 T514 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 324772 T514 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:38017_",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:38017"}
[junit4:junit4] 2> 324785 T520 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 324785 T549 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 324785 T533 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 324785 T565 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 324785 T582 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 324785 T513 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 325224 T491 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 325226 T491 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
[junit4:junit4] 2> ASYNC NEW_CORE C576 name=collection1 org.apache.solr.core.SolrCore@3f6aa4e3 url=http://127.0.0.1:54154/collection1 node=127.0.0.1:54154_ C576_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:54154_, base_url=http://127.0.0.1:54154, leader=true}
[junit4:junit4] 2> 325235 T503 C576 P54154 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 325238 T503 C576 P54154 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1aeed8ce lockFactory=org.apache.lucene.store.NativeFSLockFactory@300cb3d5; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> 325239 T503 C576 P54154 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 325240 T503 C576 P54154 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1aeed8ce lockFactory=org.apache.lucene.store.NativeFSLockFactory@300cb3d5; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@1aeed8ce lockFactory=org.apache.lucene.store.NativeFSLockFactory@300cb3d5; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_2,generation=2}
[junit4:junit4] 2> 325240 T503 C576 P54154 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 325241 T503 C576 P54154 oass.SolrIndexSearcher.<init> Opening Searcher@6893330c main
[junit4:junit4] 2> 325241 T503 C576 P54154 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 325242 T517 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6893330c main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 325243 T503 C576 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 8
[junit4:junit4] 2> 325269 T527 C573 P54158 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 325270 T527 C573 P54158 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@584249b2 lockFactory=org.apache.lucene.store.NativeFSLockFactory@466f2973; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@584249b2 lockFactory=org.apache.lucene.store.NativeFSLockFactory@466f2973; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_3,generation=3}
[junit4:junit4] 2> 325271 T527 C573 P54158 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 325271 T527 C573 P54158 oass.SolrIndexSearcher.<init> Opening Searcher@54d83ea2 main
[junit4:junit4] 2> 325272 T527 C573 P54158 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 325273 T535 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@54d83ea2 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 325290 T527 C573 P54158 oasu.SolrCmdDistributor.distribCommit Distrib commit to:[StdNode: http://127.0.0.1:60281/collection1/, StdNode: http://127.0.0.1:40567/collection1/, StdNode: http://127.0.0.1:38017/collection1/] params:commit_end_point=true&commit=true&softCommit=false&waitSearcher=true&expungeDeletes=false
[junit4:junit4] 2> 325300 T543 C575 P40567 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 325301 T543 C575 P40567 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@2c26618f lockFactory=org.apache.lucene.store.NativeFSLockFactory@1b1f4d6f; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@2c26618f lockFactory=org.apache.lucene.store.NativeFSLockFactory@1b1f4d6f; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_3,generation=3}
[junit4:junit4] 2> 325302 T543 C575 P40567 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 325302 T543 C575 P40567 oass.SolrIndexSearcher.<init> Opening Searcher@5b09b45d main
[junit4:junit4] 2> 325303 T543 C575 P40567 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 325303 T551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5b09b45d main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 325304 T543 C575 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 4
[junit4:junit4] 2> C572_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:60281_, base_url=http://127.0.0.1:60281}
[junit4:junit4] 2> 325307 T555 C572 P60281 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> C574_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:38017_, base_url=http://127.0.0.1:38017}
[junit4:junit4] 2> 325308 T572 C574 P38017 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 325308 T555 C572 P60281 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@140962cf lockFactory=org.apache.lucene.store.NativeFSLockFactory@52b1eaed; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@140962cf lockFactory=org.apache.lucene.store.NativeFSLockFactory@52b1eaed; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_3,generation=3}
[junit4:junit4] 2> 325309 T572 C574 P38017 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@4651534f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6b6f4b51; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=BaseDirectoryWrapper(RateLimitedDirectoryWrapper(NRTCachingDirectory(org.apache.lucene.store.RAMDirectory@4651534f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6b6f4b51; maxCacheMB=0.3671875 maxMergeSizeMB=0.6201171875))),segFN=segments_3,generation=3}
[junit4:junit4] 2> 325310 T572 C574 P38017 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 325309 T555 C572 P60281 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 325310 T572 C574 P38017 oass.SolrIndexSearcher.<init> Opening Searcher@24686126 main
[junit4:junit4] 2> 325311 T555 C572 P60281 oass.SolrIndexSearcher.<init> Opening Searcher@6d40c31 main
[junit4:junit4] 2> 325311 T572 C574 P38017 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 325312 T555 C572 P60281 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 325312 T584 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@24686126 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 325312 T567 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6d40c31 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 325313 T572 C574 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 5
[junit4:junit4] 2> 325313 T555 C572 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 6
[junit4:junit4] 2> 325314 T527 C573 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 45
[junit4:junit4] 2> 325316 T491 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 325340 T528 C573 P54158 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 325343 T556 C572 P60281 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 325346 T544 C575 P40567 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 325349 T573 C574 P38017 oasc.SolrCore.execute [collection1] webapp= path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 327477 T503 C576 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1436400187424636928)} 0 26
[junit4:junit4] 2> 327496 T574 C574 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1436400187430928384&update.from=http://127.0.0.1:40567/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1436400187430928384)} 0 2
[junit4:junit4] 2> 327500 T555 C572 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=FROMLEADER&_version_=-1436400187444559872&update.from=http://127.0.0.1:54158/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1436400187444559872)} 0 1
[junit4:junit4] 2> 327501 T529 C573 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={update.distrib=TOLEADER&wt=javabin&version=2} {deleteByQuery=*:* (-1436400187444559872)} 0 7
[junit4:junit4] 2> 327502 T545 C575 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1436400187430928384)} 0 22
[junit4:junit4] 2> 327508 T503 C576 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[0 (1436400187458191360)]} 0 1
[junit4:junit4] 2> 327529 T555 C572 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[0 (1436400187463434240)]} 0 1
[junit4:junit4] 2> 327530 T527 C573 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[0 (1436400187463434240)]} 0 19
[junit4:junit4] 2> 327534 T503 C576 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[1 (1436400187486502912)]} 0 1
[junit4:junit4] 2> 327551 T555 C572 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[1 (1436400187500134400)]} 0 0
[junit4:junit4] 2> 327552 T529 C573 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[1 (1436400187500134400)]} 0 5
[junit4:junit4] 2> 327553 T545 C575 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[1]} 0 16
[junit4:junit4] 2> 327557 T503 C576 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[2 (1436400187510620160)]} 0 1
[junit4:junit4] 2> 327565 T574 C574 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[2 (1436400187514814464)]} 0 0
[junit4:junit4] 2> 327566 T545 C575 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[2 (1436400187514814464)]} 0 6
[junit4:junit4] 2> 327570 T503 C576 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[3 (1436400187524251648)]} 0 1
[junit4:junit4] 2> 327577 T574 C574 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[3 (1436400187528445952)]} 0 0
[junit4:junit4] 2> 327578 T545 C575 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[3 (1436400187528445952)]} 0 5
[junit4:junit4] 2> 327582 T503 C576 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[4 (1436400187536834560)]} 0 0
[junit4:junit4] 2> 327590 T555 C572 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[4 (1436400187541028864)]} 0 0
[junit4:junit4] 2> 327591 T527 C573 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[4 (1436400187541028864)]} 0 6
[junit4:junit4] 2> 327595 T503 C576 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[5 (1436400187550466048)]} 0 1
[junit4:junit4] 2> 327602 T574 C574 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[5 (1436400187554660352)]} 0 0
[junit4:junit4] 2> 327603 T545 C575 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[5 (1436400187554660352)]} 0 5
[junit4:junit4] 2> 327607 T503 C576 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[6 (1436400187563048960)]} 0 1
[junit4:junit4] 2> ASYNC NEW_CORE C577 name=collection1 org.apache.solr.core.SolrCore@17002d90 url=http://127.0.0.1:38017/collection1 node=127.0.0.1:38017_ C577_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:38017_, base_url=http://127.0.0.1:38017}
[junit4:junit4] 2> 327623 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[6 (1436400187567243264)]} 0 8
[junit4:junit4] 2> ASYNC NEW_CORE C578 name=collection1 org.apache.solr.core.SolrCore@765bf112 url=http://127.0.0.1:40567/collection1 node=127.0.0.1:40567_ C578_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:40567_, base_url=http://127.0.0.1:40567, leader=true}
[junit4:junit4] 2> 327624 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[6 (1436400187567243264)]} 0 14
[junit4:junit4] 2> ASYNC NEW_CORE C579 name=collection1 org.apache.solr.core.SolrCore@3f6aa4e3 url=http://127.0.0.1:54154/collection1 node=127.0.0.1:54154_ C579_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:54154_, base_url=http://127.0.0.1:54154, leader=true}
[junit4:junit4] 2> 327630 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[7 (1436400187586117632)]} 0 1
[junit4:junit4] 2> 327638 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[7 (1436400187591360512)]} 0 0
[junit4:junit4] 2> 327639 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[7 (1436400187591360512)]} 0 5
[junit4:junit4] 2> 327643 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[8 (1436400187600797696)]} 0 1
[junit4:junit4] 2> ASYNC NEW_CORE C580 name=collection1 org.apache.solr.core.SolrCore@3ef99bfd url=http://127.0.0.1:60281/collection1 node=127.0.0.1:60281_ C580_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:60281_, base_url=http://127.0.0.1:60281}
[junit4:junit4] 2> 327651 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[8 (1436400187604992000)]} 0 1
[junit4:junit4] 2> ASYNC NEW_CORE C581 name=collection1 org.apache.solr.core.SolrCore@69b82b30 url=http://127.0.0.1:54158/collection1 node=127.0.0.1:54158_ C581_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:54158_, base_url=http://127.0.0.1:54158, leader=true}
[junit4:junit4] 2> 327652 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[8 (1436400187604992000)]} 0 5
[junit4:junit4] 2> 327656 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[9 (1436400187614429184)]} 0 1
[junit4:junit4] 2> 327669 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[9 (1436400187622817792)]} 0 1
[junit4:junit4] 2> 327670 T543 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[9 (1436400187622817792)]} 0 6
[junit4:junit4] 2> 327670 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[9]} 0 10
[junit4:junit4] 2> 327674 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[10 (1436400187633303552)]} 0 1
[junit4:junit4] 2> 327686 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[10 (1436400187640643584)]} 0 1
[junit4:junit4] 2> 327687 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[10 (1436400187640643584)]} 0 6
[junit4:junit4] 2> 327688 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[10]} 0 10
[junit4:junit4] 2> 327693 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[11 (1436400187653226496)]} 0 0
[junit4:junit4] 2> 327702 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[11 (1436400187658469376)]} 0 0
[junit4:junit4] 2> 327703 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[11 (1436400187658469376)]} 0 5
[junit4:junit4] 2> 327707 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[12 (1436400187667906560)]} 0 1
[junit4:junit4] 2> 327718 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[12 (1436400187675246592)]} 0 0
[junit4:junit4] 2> 327719 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[12 (1436400187675246592)]} 0 5
[junit4:junit4] 2> 327720 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[12]} 0 10
[junit4:junit4] 2> 327724 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[13 (1436400187685732352)]} 0 1
[junit4:junit4] 2> 327732 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[13 (1436400187689926656)]} 0 1
[junit4:junit4] 2> 327733 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[13 (1436400187689926656)]} 0 6
[junit4:junit4] 2> 327737 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[14 (1436400187699363840)]} 0 1
[junit4:junit4] 2> 327744 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[14 (1436400187703558144)]} 0 0
[junit4:junit4] 2> 327745 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[14 (1436400187703558144)]} 0 5
[junit4:junit4] 2> 327749 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[15 (1436400187711946752)]} 0 1
[junit4:junit4] 2> 327756 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[15 (1436400187716141056)]} 0 0
[junit4:junit4] 2> 327757 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[15 (1436400187716141056)]} 0 5
[junit4:junit4] 2> 327761 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[16 (1436400187724529664)]} 0 1
[junit4:junit4] 2> 327768 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[16 (1436400187728723968)]} 0 0
[junit4:junit4] 2> 327769 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[16 (1436400187728723968)]} 0 5
[junit4:junit4] 2> 327773 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[17 (1436400187737112576)]} 0 1
[junit4:junit4] 2> 327781 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[17 (1436400187741306880)]} 0 1
[junit4:junit4] 2> 327782 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[17 (1436400187741306880)]} 0 6
[junit4:junit4] 2> 327786 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[18 (1436400187750744064)]} 0 1
[junit4:junit4] 2> 327796 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[18 (1436400187758084096)]} 0 0
[junit4:junit4] 2> 327797 T543 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[18 (1436400187758084096)]} 0 4
[junit4:junit4] 2> 327798 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[18]} 0 9
[junit4:junit4] 2> 327802 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[19 (1436400187767521280)]} 0 1
[junit4:junit4] 2> 327810 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[19 (1436400187772764160)]} 0 0
[junit4:junit4] 2> 327811 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[19 (1436400187772764160)]} 0 5
[junit4:junit4] 2> 327816 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[20 (1436400187781152768)]} 0 1
[junit4:junit4] 2> 327823 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[20 (1436400187786395648)]} 0 0
[junit4:junit4] 2> 327824 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[20 (1436400187786395648)]} 0 5
[junit4:junit4] 2> 327828 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[21 (1436400187794784256)]} 0 0
[junit4:junit4] 2> 327839 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[21 (1436400187802124288)]} 0 0
[junit4:junit4] 2> 327840 T543 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[21 (1436400187802124288)]} 0 5
[junit4:junit4] 2> 327841 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[21]} 0 10
[junit4:junit4] 2> 327845 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[22 (1436400187812610048)]} 0 1
[junit4:junit4] 2> 327853 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[22 (1436400187816804352)]} 0 1
[junit4:junit4] 2> 327854 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[22 (1436400187816804352)]} 0 6
[junit4:junit4] 2> 327858 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[23 (1436400187826241536)]} 0 1
[junit4:junit4] 2> 327866 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[23 (1436400187830435840)]} 0 0
[junit4:junit4] 2> 327867 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[23 (1436400187830435840)]} 0 6
[junit4:junit4] 2> 327871 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[24 (1436400187839873024)]} 0 1
[junit4:junit4] 2> 327881 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[24 (1436400187847213056)]} 0 1
[junit4:junit4] 2> 327882 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[24 (1436400187847213056)]} 0 5
[junit4:junit4] 2> 327883 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[24]} 0 9
[junit4:junit4] 2> 327886 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[25 (1436400187855601664)]} 0 1
[junit4:junit4] 2> 327893 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[25 (1436400187859795968)]} 0 1
[junit4:junit4] 2> 327894 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[25 (1436400187859795968)]} 0 5
[junit4:junit4] 2> 327898 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[26 (1436400187868184576)]} 0 1
[junit4:junit4] 2> 327909 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[26 (1436400187875524608)]} 0 1
[junit4:junit4] 2> 327910 T543 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[26 (1436400187875524608)]} 0 5
[junit4:junit4] 2> 327911 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[26]} 0 10
[junit4:junit4] 2> 327915 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[27 (1436400187884961792)]} 0 1
[junit4:junit4] 2> 327925 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[27 (1436400187893350400)]} 0 1
[junit4:junit4] 2> 327926 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[27 (1436400187893350400)]} 0 5
[junit4:junit4] 2> 327927 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[27]} 0 8
[junit4:junit4] 2> 327931 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[28 (1436400187901739008)]} 0 1
[junit4:junit4] 2> 327941 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[28 (1436400187910127616)]} 0 1
[junit4:junit4] 2> 327942 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[28 (1436400187910127616)]} 0 5
[junit4:junit4] 2> 327943 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[28]} 0 9
[junit4:junit4] 2> 327947 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[29 (1436400187919564800)]} 0 1
[junit4:junit4] 2> 327954 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[29 (1436400187923759104)]} 0 1
[junit4:junit4] 2> 327955 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[29 (1436400187923759104)]} 0 5
[junit4:junit4] 2> 327959 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[30 (1436400187932147712)]} 0 1
[junit4:junit4] 2> 327969 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[30 (1436400187939487744)]} 0 1
[junit4:junit4] 2> 327970 T543 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[30 (1436400187939487744)]} 0 5
[junit4:junit4] 2> 327970 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[30]} 0 8
[junit4:junit4] 2> 327974 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[31 (1436400187946827776)]} 0 1
[junit4:junit4] 2> 327980 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[31 (1436400187949973504)]} 0 1
[junit4:junit4] 2> 327981 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[31 (1436400187949973504)]} 0 4
[junit4:junit4] 2> 327984 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[32 (1436400187957313536)]} 0 1
[junit4:junit4] 2> 327995 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[32 (1436400187965702144)]} 0 1
[junit4:junit4] 2> 327996 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[32 (1436400187965702144)]} 0 6
[junit4:junit4] 2> 327996 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[32]} 0 9
[junit4:junit4] 2> 328000 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[33 (1436400187974090752)]} 0 1
[junit4:junit4] 2> 328006 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[33 (1436400187977236480)]} 0 1
[junit4:junit4] 2> 328007 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[33 (1436400187977236480)]} 0 5
[junit4:junit4] 2> 328009 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[34 (1436400187984576512)]} 0 0
[junit4:junit4] 2> 328019 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[34 (1436400187990867968)]} 0 1
[junit4:junit4] 2> 328019 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[34 (1436400187990867968)]} 0 4
[junit4:junit4] 2> 328020 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[34]} 0 8
[junit4:junit4] 2> 328024 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[35 (1436400187999256576)]} 0 1
[junit4:junit4] 2> 328033 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[35 (1436400188006596608)]} 0 1
[junit4:junit4] 2> 328033 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[35 (1436400188006596608)]} 0 4
[junit4:junit4] 2> 328034 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[35]} 0 8
[junit4:junit4] 2> 328038 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[36 (1436400188013936640)]} 0 1
[junit4:junit4] 2> 328044 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[36 (1436400188017082368)]} 0 1
[junit4:junit4] 2> 328044 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[36 (1436400188017082368)]} 0 4
[junit4:junit4] 2> 328048 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[37 (1436400188024422400)]} 0 1
[junit4:junit4] 2> 328055 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[37 (1436400188029665280)]} 0 1
[junit4:junit4] 2> 328055 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[37 (1436400188029665280)]} 0 4
[junit4:junit4] 2> 328058 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[38 (1436400188035956736)]} 0 0
[junit4:junit4] 2> 328068 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[38 (1436400188043296768)]} 0 1
[junit4:junit4] 2> 328069 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[38 (1436400188043296768)]} 0 5
[junit4:junit4] 2> 328069 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[38]} 0 8
[junit4:junit4] 2> 328073 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[39 (1436400188050636800)]} 0 1
[junit4:junit4] 2> 328079 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[39 (1436400188053782528)]} 0 1
[junit4:junit4] 2> 328080 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[39 (1436400188053782528)]} 0 5
[junit4:junit4] 2> 328084 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[40 (1436400188063219712)]} 0 1
[junit4:junit4] 2> 328091 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[40 (1436400188066365440)]} 0 1
[junit4:junit4] 2> 328092 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[40 (1436400188066365440)]} 0 5
[junit4:junit4] 2> 328096 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[41 (1436400188075802624)]} 0 0
[junit4:junit4] 2> 328106 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[41 (1436400188083142656)]} 0 0
[junit4:junit4] 2> 328107 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[41 (1436400188083142656)]} 0 5
[junit4:junit4] 2> 328108 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[41]} 0 9
[junit4:junit4] 2> 328112 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[42 (1436400188091531264)]} 0 1
[junit4:junit4] 2> 328123 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[42 (1436400188099919872)]} 0 1
[junit4:junit4] 2> 328123 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[42 (1436400188099919872)]} 0 4
[junit4:junit4] 2> 328124 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[42]} 0 9
[junit4:junit4] 2> 328128 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[43 (1436400188108308480)]} 0 1
[junit4:junit4] 2> 328137 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[43 (1436400188115648512)]} 0 1
[junit4:junit4] 2> 328137 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[43 (1436400188115648512)]} 0 4
[junit4:junit4] 2> 328138 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[43]} 0 8
[junit4:junit4] 2> 328142 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[44 (1436400188122988544)]} 0 1
[junit4:junit4] 2> 328152 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[44 (1436400188131377152)]} 0 1
[junit4:junit4] 2> 328152 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[44 (1436400188131377152)]} 0 4
[junit4:junit4] 2> 328153 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[44]} 0 7
[junit4:junit4] 2> 328157 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[45 (1436400188138717184)]} 0 1
[junit4:junit4] 2> 328166 T574 C577 P38017 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[45 (1436400188145008640)]} 0 1
[junit4:junit4] 2> 328167 T543 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[45 (1436400188145008640)]} 0 5
[junit4:junit4] 2> 328168 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[45]} 0 9
[junit4:junit4] 2> 328171 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[46 (1436400188154445824)]} 0 1
[junit4:junit4] 2> 328181 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[46 (1436400188161785856)]} 0 1
[junit4:junit4] 2> 328182 T529 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:40567/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[46 (1436400188161785856)]} 0 5
[junit4:junit4] 2> 328183 T545 C578 P40567 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[46]} 0 9
[junit4:junit4] 2> 328186 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[47 (1436400188170174464)]} 0 1
[junit4:junit4] 2> 328193 T555 C580 P60281 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={distrib.from=http://127.0.0.1:54158/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[47 (1436400188173320192)]} 0 1
[junit4:junit4] 2> 328194 T527 C581 P54158 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[47 (1436400188173320192)]} 0 5
[junit4:junit4] 2> 328198 T503 C579 P54154 oasup.LogUpdateProcessor.finish [collection1] webapp= p
[...truncated too long message...]
dPoolExecutor.java:1146)
[junit4:junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4:junit4] 2> at java.lang.Thread.run(Thread.java:679)
[junit4:junit4] 2> NOTE: test params are: codec=Lucene42: {timestamp=PostingsFormat(name=Memory doPackFST= false), range_facet_l=PostingsFormat(name=Memory doPackFST= false), id=Pulsing41(freqCutoff=2 minBlockSize=66 maxBlockSize=175), range_facet_sl=Pulsing41(freqCutoff=6 minBlockSize=66 maxBlockSize=175), a_t=PostingsFormat(name=Memory doPackFST= false), text=Pulsing41(freqCutoff=6 minBlockSize=66 maxBlockSize=175), range_facet_si=Lucene41(blocksize=128), _version_=PostingsFormat(name=Memory doPackFST= false), other_tl1=PostingsFormat(name=Memory doPackFST= false), multiDefault=Lucene41(blocksize=128), a_si=Lucene41(blocksize=128), intDefault=PostingsFormat(name=Memory doPackFST= false)}, docValues:{}, sim=DefaultSimilarity, locale=iw_IL, timezone=America/Argentina/Rio_Gallegos
[junit4:junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Sun Microsystems Inc. 1.6.0_32 (64-bit)/cpus=16,threads=2,free=114958632,total=478478336
[junit4:junit4] 2> NOTE: All tests run in this JVM: [RequiredFieldsTest, SampleTest, TestRandomFaceting, ShardRoutingCustomTest, TestOmitPositions, CopyFieldTest, TestLuceneMatchVersion, TestCopyFieldCollectionResource, HighlighterTest, TestSolr4Spatial, CachingDirectoryFactoryTest, DirectUpdateHandlerTest, TestUpdate, TestPropInject, IndexReaderFactoryTest, TestFoldingMultitermQuery, XsltUpdateRequestHandlerTest, ChaosMonkeyShardSplitTest, FieldAnalysisRequestHandlerTest, TestAddFieldRealTimeGet, MultiTermTest, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, TestNumberUtils, DOMUtilTest, TestAtomicUpdateErrorCases, URLClassifyProcessorTest, TestZkChroot, TestShardHandlerFactory, OutputWriterTest, TestCSVLoader, SimplePostToolTest, LeaderElectionIntegrationTest, TestRecovery, XmlUpdateRequestHandlerTest, EchoParamsTest, DocumentAnalysisRequestHandlerTest, DisMaxRequestHandlerTest, DateMathParserTest, SolrTestCaseJ4Test, SpellCheckCollatorTest, TestIndexSearcher, TestLMJelinekMercerSimilarityFactory, FileUtilsTest, UniqFieldsUpdateProcessorFactoryTest, ZkSolrClientTest, TestWordDelimiterFilterFactory, TestPostingsSolrHighlighter, TestReload, TestSchemaNameResource, TestDynamicFieldCollectionResource, TestSolrXml, CoreContainerCoreInitFailuresTest, TestSchemaSimilarityResource, TestUtils, NumericFieldsTest, NotRequiredUniqueKeyTest, SuggesterTSTTest, TestDistributedSearch, TestSolrQueryParserDefaultOperatorResource, TestCoreDiscovery, TestUniqueKeyFieldResource, HighlighterConfigTest, BadIndexSchemaTest, SignatureUpdateProcessorFactoryTest, RequestHandlersTest, BasicDistributedZk2Test, DirectSolrSpellCheckerTest, TestHashPartitioner, FullSolrCloudDistribCmdsTest, CollectionsAPIDistributedZkTest]
[junit4:junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.seed=C92FB91E9C81331 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/hudson/lucene-data/enwiki.random.lines.txt -Dtests.locale=iw_IL -Dtests.timezone=America/Argentina/Rio_Gallegos -Dtests.file.encoding=ISO-8859-1
[junit4:junit4] ERROR 0.00s J1 | CollectionsAPIDistributedZkTest (suite) <<<
[junit4:junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:
[junit4:junit4] > 1) Thread[id=2887, name=recoveryCmdExecutor-1223-thread-1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
[junit4:junit4] > at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4:junit4] > at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:327)
[junit4:junit4] > at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:193)
[junit4:junit4] > at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:180)
[junit4:junit4] > at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:384)
[junit4:junit4] > at java.net.Socket.connect(Socket.java:546)
[junit4:junit4] > at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
[junit4:junit4] > at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
[junit4:junit4] > at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
[junit4:junit4] > at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
[junit4:junit4] > at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
[junit4:junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
[junit4:junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
[junit4:junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4:junit4] > at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:298)
[junit4:junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
[junit4:junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:679)
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([C92FB91E9C81331]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4:junit4] > 1) Thread[id=2887, name=recoveryCmdExecutor-1223-thread-1, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
[junit4:junit4] > at java.net.PlainSocketImpl.socketConnect(Native Method)
[junit4:junit4] > at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:327)
[junit4:junit4] > at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:193)
[junit4:junit4] > at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:180)
[junit4:junit4] > at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:384)
[junit4:junit4] > at java.net.Socket.connect(Socket.java:546)
[junit4:junit4] > at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:127)
[junit4:junit4] > at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
[junit4:junit4] > at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:294)
[junit4:junit4] > at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:645)
[junit4:junit4] > at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:480)
[junit4:junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:906)
[junit4:junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:805)
[junit4:junit4] > at org.apache.http.impl.client.AbstractHttpClient.execute(AbstractHttpClient.java:784)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:365)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4:junit4] > at org.apache.solr.cloud.SyncStrategy$1.run(SyncStrategy.java:298)
[junit4:junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
[junit4:junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:679)
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([C92FB91E9C81331]:0)
[junit4:junit4] Completed on J1 in 362.90s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 458 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:393: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:366: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/build.xml:181: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/solr/common-build.xml:437: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:1243: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-NightlyTests-4.x/lucene/common-build.xml:887: There were test failures: 296 suites, 1235 tests, 2 suite-level errors, 1 error, 1 failure, 566 ignored (3 assumptions)
Total time: 90 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure