You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2013/05/18 12:24:07 UTC
[JENKINS] Lucene-Solr-Tests-4.x-java7 - Build # 1260 - Failure
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-java7/1260/
1 tests failed.
REGRESSION: org.apache.solr.cloud.ChaosMonkeyShardSplitTest.testDistribSearch
Error Message:
Wrong doc count on shard1_1 expected:<400> but was:<399>
Stack Trace:
java.lang.AssertionError: Wrong doc count on shard1_1 expected:<400> but was:<399>
at __randomizedtesting.SeedInfo.seed([74BCBF895EB06A15:F55A319129EF0A29]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.ShardSplitTest.checkDocCountsAndShardStates(ShardSplitTest.java:167)
at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:155)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:722)
Build Log:
[...truncated 9289 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeyShardSplitTest
[junit4:junit4] 2> 91067 T241 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /m_wd/c
[junit4:junit4] 2> 91072 T241 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./solrtest-ChaosMonkeyShardSplitTest-1368871031133
[junit4:junit4] 2> 91074 T241 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 91074 T242 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 91175 T241 oasc.ZkTestServer.run start zk server on port:15823
[junit4:junit4] 2> 91178 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 91182 T248 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e326437 name:ZooKeeperConnection Watcher:127.0.0.1:15823 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 91182 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 91182 T241 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 91201 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 91203 T250 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bea6d2d name:ZooKeeperConnection Watcher:127.0.0.1:15823/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 91203 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 91203 T241 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 91212 T241 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 91215 T241 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 91217 T241 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 91227 T241 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 91227 T241 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 91233 T241 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 91234 T241 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 91337 T241 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 91338 T241 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 91341 T241 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 91342 T241 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 91345 T241 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 91346 T241 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 91349 T241 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 91349 T241 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 91352 T241 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 91353 T241 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 91361 T241 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 91361 T241 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 91364 T241 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 91365 T241 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 91529 T241 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 91531 T241 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:15826
[junit4:junit4] 2> 91532 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 91532 T241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 91533 T241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368871031435
[junit4:junit4] 2> 91533 T241 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368871031435/solr.xml
[junit4:junit4] 2> 91533 T241 oasc.CoreContainer.<init> New CoreContainer 1294425435
[junit4:junit4] 2> 91534 T241 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368871031435/'
[junit4:junit4] 2> 91534 T241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368871031435/'
[junit4:junit4] 2> 91625 T241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 91626 T241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 91626 T241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 91626 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 91627 T241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 91627 T241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 91627 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 91628 T241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 91628 T241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 91628 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 91640 T241 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 91641 T241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:15823/solr
[junit4:junit4] 2> 91641 T241 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 91642 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 91646 T261 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e07208 name:ZooKeeperConnection Watcher:127.0.0.1:15823 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 91647 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 91649 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 91659 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 91661 T263 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c96792b name:ZooKeeperConnection Watcher:127.0.0.1:15823/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 91661 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 91663 T241 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 91673 T241 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 91677 T241 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 91680 T241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:15826_m_wd%2Fc
[junit4:junit4] 2> 91693 T241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:15826_m_wd%2Fc
[junit4:junit4] 2> 91697 T241 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 91709 T241 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 91718 T241 oasc.Overseer.start Overseer (id=89710331896659971-127.0.0.1:15826_m_wd%2Fc-n_0000000000) starting
[junit4:junit4] 2> 91723 T241 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 91737 T265 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 91737 T241 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 91740 T241 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 91748 T241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 91751 T264 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 91754 T266 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368871031435/collection1
[junit4:junit4] 2> 91754 T266 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 91755 T266 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 91755 T266 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 91756 T266 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368871031435/collection1/'
[junit4:junit4] 2> 91757 T266 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368871031435/collection1/lib/README' to classloader
[junit4:junit4] 2> 91758 T266 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368871031435/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 91797 T266 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 91852 T266 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 91954 T266 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 91962 T266 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 92574 T266 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 92580 T266 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 92583 T266 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 92590 T266 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 92620 T266 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 92621 T266 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368871031435/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/control/data/
[junit4:junit4] 2> 92621 T266 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5122fd03
[junit4:junit4] 2> 92622 T266 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 92622 T266 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/control/data
[junit4:junit4] 2> 92623 T266 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/control/data/index/
[junit4:junit4] 2> 92623 T266 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 92623 T266 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/control/data/index
[junit4:junit4] 2> 92628 T266 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5e8e11d5 lockFactory=org.apache.lucene.store.NativeFSLockFactory@11d75ba),segFN=segments_1,generation=1}
[junit4:junit4] 2> 92628 T266 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 92694 T266 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 92695 T266 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 92695 T266 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 92696 T266 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 92697 T266 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 92697 T266 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 92697 T266 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 92698 T266 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 92698 T266 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 92713 T266 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 92720 T266 oass.SolrIndexSearcher.<init> Opening Searcher@1f3de057 main
[junit4:junit4] 2> 92720 T266 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/control/data/tlog
[junit4:junit4] 2> 92721 T266 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 92722 T266 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 92726 T267 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1f3de057 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 92728 T266 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 92728 T266 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 93255 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 93256 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:15826_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:15826/m_wd/c"}
[junit4:junit4] 2> 93257 T264 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 93263 T264 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 93278 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 93732 T266 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 93732 T266 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:15826/m_wd/c collection:control_collection shard:shard1
[junit4:junit4] 2> 93733 T266 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 93752 T266 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 93762 T266 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 93763 T266 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 93763 T266 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:15826/m_wd/c/collection1/
[junit4:junit4] 2> 93763 T266 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 93763 T266 oasc.SyncStrategy.syncToMe http://127.0.0.1:15826/m_wd/c/collection1/ has no replicas
[junit4:junit4] 2> 93763 T266 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:15826/m_wd/c/collection1/
[junit4:junit4] 2> 93764 T266 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 94783 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 94832 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 94842 T266 oasc.ZkController.register We are http://127.0.0.1:15826/m_wd/c/collection1/ and leader is http://127.0.0.1:15826/m_wd/c/collection1/
[junit4:junit4] 2> 94842 T266 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:15826/m_wd/c
[junit4:junit4] 2> 94842 T266 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 94843 T266 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 94843 T266 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 94845 T266 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 94847 T241 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 94847 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 94926 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 94955 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 94962 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 94964 T270 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c3e2537 name:ZooKeeperConnection Watcher:127.0.0.1:15823/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 94965 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 94967 T241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 94972 T241 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 95187 T241 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 95190 T241 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20726
[junit4:junit4] 2> 95191 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 95191 T241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 95192 T241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368871035033
[junit4:junit4] 2> 95192 T241 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368871035033/solr.xml
[junit4:junit4] 2> 95193 T241 oasc.CoreContainer.<init> New CoreContainer 736842717
[junit4:junit4] 2> 95193 T241 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368871035033/'
[junit4:junit4] 2> 95194 T241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368871035033/'
[junit4:junit4] 2> 95322 T241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 95323 T241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 95323 T241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 95323 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 95324 T241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 95324 T241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 95325 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 95325 T241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 95326 T241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 95326 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 95344 T241 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 95345 T241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:15823/solr
[junit4:junit4] 2> 95345 T241 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 95347 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 95350 T281 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@592a1fe3 name:ZooKeeperConnection Watcher:127.0.0.1:15823 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 95351 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 95365 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 95379 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 95382 T283 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a55bc7d name:ZooKeeperConnection Watcher:127.0.0.1:15823/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 95382 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 95394 T241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 96340 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 96341 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:15826_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:15826/m_wd/c"}
[junit4:junit4] 2> 96390 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 96390 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 96390 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 96399 T241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20726_m_wd%2Fc
[junit4:junit4] 2> 96401 T241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20726_m_wd%2Fc
[junit4:junit4] 2> 96427 T263 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 96427 T283 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 96428 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 96427 T270 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 96428 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 96428 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 96441 T284 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368871035033/collection1
[junit4:junit4] 2> 96441 T284 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 96442 T284 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 96442 T284 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 96443 T284 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368871035033/collection1/'
[junit4:junit4] 2> 96444 T284 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368871035033/collection1/lib/README' to classloader
[junit4:junit4] 2> 96445 T284 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368871035033/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 96485 T284 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 96538 T284 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 96640 T284 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 96648 T284 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 97183 T284 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 97187 T284 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 97189 T284 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 97195 T284 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 97225 T284 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 97226 T284 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368871035033/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty1/
[junit4:junit4] 2> 97226 T284 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5122fd03
[junit4:junit4] 2> 97226 T284 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 97227 T284 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty1
[junit4:junit4] 2> 97227 T284 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty1/index/
[junit4:junit4] 2> 97227 T284 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 97228 T284 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty1/index
[junit4:junit4] 2> 97231 T284 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4abacbab lockFactory=org.apache.lucene.store.NativeFSLockFactory@48e1dcac),segFN=segments_1,generation=1}
[junit4:junit4] 2> 97231 T284 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 97233 T284 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 97234 T284 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 97234 T284 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 97235 T284 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 97235 T284 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 97236 T284 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 97236 T284 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 97237 T284 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 97237 T284 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 97248 T284 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 97254 T284 oass.SolrIndexSearcher.<init> Opening Searcher@1f3755e0 main
[junit4:junit4] 2> 97254 T284 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty1/tlog
[junit4:junit4] 2> 97255 T284 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 97255 T284 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 97260 T285 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1f3755e0 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 97262 T284 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 97262 T284 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 97897 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 97898 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:20726_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:20726/m_wd/c"}
[junit4:junit4] 2> 97899 T264 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4:junit4] 2> 97899 T264 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 97930 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 97930 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 97930 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 98290 T284 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 98290 T284 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20726/m_wd/c collection:collection1 shard:shard1
[junit4:junit4] 2> 98291 T284 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 98338 T284 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 98342 T284 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 98342 T284 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 98342 T284 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20726/m_wd/c/collection1/
[junit4:junit4] 2> 98343 T284 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 98343 T284 oasc.SyncStrategy.syncToMe http://127.0.0.1:20726/m_wd/c/collection1/ has no replicas
[junit4:junit4] 2> 98343 T284 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20726/m_wd/c/collection1/
[junit4:junit4] 2> 98344 T284 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 99443 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 99471 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 99472 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 99472 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 99481 T284 oasc.ZkController.register We are http://127.0.0.1:20726/m_wd/c/collection1/ and leader is http://127.0.0.1:20726/m_wd/c/collection1/
[junit4:junit4] 2> 99481 T284 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20726/m_wd/c
[junit4:junit4] 2> 99481 T284 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 99481 T284 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 99482 T284 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 99484 T284 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 99485 T241 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 99486 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 99486 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 99702 T241 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 99705 T241 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:16736
[junit4:junit4] 2> 99706 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 99706 T241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 99706 T241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368871039560
[junit4:junit4] 2> 99707 T241 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368871039560/solr.xml
[junit4:junit4] 2> 99707 T241 oasc.CoreContainer.<init> New CoreContainer 648392891
[junit4:junit4] 2> 99708 T241 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368871039560/'
[junit4:junit4] 2> 99708 T241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368871039560/'
[junit4:junit4] 2> 99816 T241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 99817 T241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 99817 T241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 99818 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 99818 T241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 99818 T241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 99819 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 99819 T241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 99820 T241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 99820 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 99834 T241 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 99834 T241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:15823/solr
[junit4:junit4] 2> 99835 T241 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 99836 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 99840 T297 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5aaf6fff name:ZooKeeperConnection Watcher:127.0.0.1:15823 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 99840 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 99853 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 99863 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 99865 T299 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@50cce66c name:ZooKeeperConnection Watcher:127.0.0.1:15823/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 99866 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 99876 T241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 100880 T241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:16736_m_wd%2Fc
[junit4:junit4] 2> 100883 T241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:16736_m_wd%2Fc
[junit4:junit4] 2> 100887 T263 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 100887 T283 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 100887 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 100887 T299 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 100887 T270 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 100887 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 100888 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 100906 T300 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368871039560/collection1
[junit4:junit4] 2> 100906 T300 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 100907 T300 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 100907 T300 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 100919 T300 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368871039560/collection1/'
[junit4:junit4] 2> 100921 T300 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368871039560/collection1/lib/README' to classloader
[junit4:junit4] 2> 100922 T300 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368871039560/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 100982 T300 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 100985 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 100987 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:20726_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:20726/m_wd/c"}
[junit4:junit4] 2> 101000 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 101000 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 101000 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 101000 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 101051 T300 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 101152 T300 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 101162 T300 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 101777 T300 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 101782 T300 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 101785 T300 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 101791 T300 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 101823 T300 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 101823 T300 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368871039560/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty2/
[junit4:junit4] 2> 101824 T300 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5122fd03
[junit4:junit4] 2> 101824 T300 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 101825 T300 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty2
[junit4:junit4] 2> 101825 T300 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty2/index/
[junit4:junit4] 2> 101826 T300 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 101826 T300 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty2/index
[junit4:junit4] 2> 101829 T300 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@fdc019b lockFactory=org.apache.lucene.store.NativeFSLockFactory@145a438c),segFN=segments_1,generation=1}
[junit4:junit4] 2> 101830 T300 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 101833 T300 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 101833 T300 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 101834 T300 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 101835 T300 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 101836 T300 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 101836 T300 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 101836 T300 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 101837 T300 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 101838 T300 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 101850 T300 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 101857 T300 oass.SolrIndexSearcher.<init> Opening Searcher@2daf1b1d main
[junit4:junit4] 2> 101857 T300 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty2/tlog
[junit4:junit4] 2> 101858 T300 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 101858 T300 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 101864 T301 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2daf1b1d main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 101867 T300 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 101867 T300 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 102507 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 102509 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:16736_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:16736/m_wd/c"}
[junit4:junit4] 2> 102509 T264 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 102509 T264 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 102522 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 102522 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 102522 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 102522 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 102870 T300 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 102870 T300 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:16736/m_wd/c collection:collection1 shard:shard2
[junit4:junit4] 2> 102871 T300 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 102897 T300 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 102909 T300 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 102909 T300 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 102909 T300 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:16736/m_wd/c/collection1/
[junit4:junit4] 2> 102910 T300 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 102910 T300 oasc.SyncStrategy.syncToMe http://127.0.0.1:16736/m_wd/c/collection1/ has no replicas
[junit4:junit4] 2> 102910 T300 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:16736/m_wd/c/collection1/
[junit4:junit4] 2> 102911 T300 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 104028 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 104093 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 104093 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 104093 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 104093 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 104151 T300 oasc.ZkController.register We are http://127.0.0.1:16736/m_wd/c/collection1/ and leader is http://127.0.0.1:16736/m_wd/c/collection1/
[junit4:junit4] 2> 104151 T300 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:16736/m_wd/c
[junit4:junit4] 2> 104151 T300 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 104151 T300 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 104152 T300 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 104199 T300 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 104200 T241 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 104201 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 104202 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 104442 T241 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 104445 T241 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:60912
[junit4:junit4] 2> 104446 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 104447 T241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 104447 T241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368871044282
[junit4:junit4] 2> 104448 T241 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368871044282/solr.xml
[junit4:junit4] 2> 104448 T241 oasc.CoreContainer.<init> New CoreContainer 1063189925
[junit4:junit4] 2> 104449 T241 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368871044282/'
[junit4:junit4] 2> 104449 T241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368871044282/'
[junit4:junit4] 2> 104575 T241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 104576 T241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 104577 T241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 104577 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 104578 T241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 104578 T241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 104579 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 104579 T241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 104580 T241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 104580 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 104598 T241 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 104599 T241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:15823/solr
[junit4:junit4] 2> 104599 T241 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 104600 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 104605 T313 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@506d4f31 name:ZooKeeperConnection Watcher:127.0.0.1:15823 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 104606 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 104609 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 104624 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 104656 T315 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e76b039 name:ZooKeeperConnection Watcher:127.0.0.1:15823/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 104656 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 104696 T241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 105700 T241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60912_m_wd%2Fc
[junit4:junit4] 2> 105702 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 105706 T241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60912_m_wd%2Fc
[junit4:junit4] 2> 105706 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:16736_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:16736/m_wd/c"}
[junit4:junit4] 2> 105716 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 105717 T283 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 105717 T270 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 105717 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 105717 T315 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 105717 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 105720 T315 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 105721 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 105721 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 105729 T299 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 105729 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 105733 T263 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 105733 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 105738 T316 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368871044282/collection1
[junit4:junit4] 2> 105739 T316 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 105741 T316 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 105741 T316 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 105742 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 105743 T316 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368871044282/collection1/'
[junit4:junit4] 2> 105745 T316 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368871044282/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 105746 T316 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368871044282/collection1/lib/README' to classloader
[junit4:junit4] 2> 105802 T316 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 105879 T316 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 105981 T316 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 105989 T316 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 106660 T316 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 106664 T316 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 106667 T316 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 106673 T316 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 106705 T316 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 106705 T316 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368871044282/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/
[junit4:junit4] 2> 106706 T316 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5122fd03
[junit4:junit4] 2> 106706 T316 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 106707 T316 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3
[junit4:junit4] 2> 106707 T316 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/index/
[junit4:junit4] 2> 106708 T316 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 106708 T316 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/index
[junit4:junit4] 2> 106712 T316 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@f189a9f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6cf024e),segFN=segments_1,generation=1}
[junit4:junit4] 2> 106712 T316 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 106716 T316 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 106716 T316 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 106717 T316 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 106718 T316 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 106718 T316 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 106718 T316 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 106719 T316 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 106720 T316 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 106720 T316 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 106735 T316 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 106742 T316 oass.SolrIndexSearcher.<init> Opening Searcher@355e64c7 main
[junit4:junit4] 2> 106742 T316 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/tlog
[junit4:junit4] 2> 106743 T316 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 106743 T316 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 106749 T317 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@355e64c7 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 106752 T316 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 106752 T316 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 107235 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 107237 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:60912_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:60912/m_wd/c"}
[junit4:junit4] 2> 107237 T264 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 107237 T264 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 107244 T315 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 107244 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 107244 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 107244 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 107244 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 107781 T316 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 107781 T316 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:60912/m_wd/c collection:collection1 shard:shard1
[junit4:junit4] 2> 107787 T316 oasc.ZkController.register We are http://127.0.0.1:60912/m_wd/c/collection1/ and leader is http://127.0.0.1:20726/m_wd/c/collection1/
[junit4:junit4] 2> 107787 T316 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60912/m_wd/c
[junit4:junit4] 2> 107787 T316 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 107787 T316 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C15 name=collection1 org.apache.solr.core.SolrCore@2b0fd239 url=http://127.0.0.1:60912/m_wd/c/collection1 node=127.0.0.1:60912_m_wd%2Fc C15_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:60912_m_wd%2Fc, base_url=http://127.0.0.1:60912/m_wd/c}
[junit4:junit4] 2> 107793 T318 C15 P60912 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 107793 T316 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 107794 T318 C15 P60912 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 107794 T318 C15 P60912 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 107795 T318 C15 P60912 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 107795 T241 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 107796 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 107797 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 107797 T318 C15 P60912 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 108074 T241 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 108077 T241 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:55117
[junit4:junit4] 2> 108078 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 108078 T241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 108079 T241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368871047877
[junit4:junit4] 2> 108079 T241 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368871047877/solr.xml
[junit4:junit4] 2> 108080 T241 oasc.CoreContainer.<init> New CoreContainer 1689952302
[junit4:junit4] 2> 108080 T241 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368871047877/'
[junit4:junit4] 2> 108081 T241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368871047877/'
[junit4:junit4] 2> 108229 T241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 108230 T241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 108230 T241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 108231 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 108231 T241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 108232 T241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 108232 T241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 108233 T241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 108233 T241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 108234 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 108253 T241 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 108254 T241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:15823/solr
[junit4:junit4] 2> 108254 T241 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 108256 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 108258 T330 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@265b6d8 name:ZooKeeperConnection Watcher:127.0.0.1:15823 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 108259 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 108262 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 108276 T241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 108279 T332 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69b98fae name:ZooKeeperConnection Watcher:127.0.0.1:15823/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 108279 T241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 108291 T241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 108750 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 108751 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:60912_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:60912/m_wd/c"}
[junit4:junit4] 2> 108768 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 108768 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 108768 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 108768 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 108768 T332 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 108768 T315 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 109000 T273 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 109001 T273 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true for: 0 seconds.
[junit4:junit4] 2> 109003 T273 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=4&state=recovering&nodeName=127.0.0.1:60912_m_wd%252Fc&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=8
[junit4:junit4] 2> 109295 T241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55117_m_wd%2Fc
[junit4:junit4] 2> 109300 T241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55117_m_wd%2Fc
[junit4:junit4] 2> 109303 T315 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 109303 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 109305 T332 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 109306 T263 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 109305 T283 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 109306 T270 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 109307 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 109307 T299 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 109307 T315 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 109307 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 109306 T332 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 109308 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 109329 T333 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368871047877/collection1
[junit4:junit4] 2> 109330 T333 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 109331 T333 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 109331 T333 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 109333 T333 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368871047877/collection1/'
[junit4:junit4] 2> 109334 T333 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368871047877/collection1/lib/README' to classloader
[junit4:junit4] 2> 109335 T333 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368871047877/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 109404 T333 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 109474 T333 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 109576 T333 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 109582 T333 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 110112 T333 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 110117 T333 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 110119 T333 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 110124 T333 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 110151 T333 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 110152 T333 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368871047877/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/
[junit4:junit4] 2> 110153 T333 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5122fd03
[junit4:junit4] 2> 110154 T333 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 110154 T333 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4
[junit4:junit4] 2> 110155 T333 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index/
[junit4:junit4] 2> 110155 T333 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 110156 T333 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index
[junit4:junit4] 2> 110159 T333 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39900ea1 lockFactory=org.apache.lucene.store.NativeFSLockFactory@15863722),segFN=segments_1,generation=1}
[junit4:junit4] 2> 110160 T333 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 110163 T333 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 110164 T333 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 110165 T333 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 110166 T333 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 110167 T333 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 110167 T333 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 110168 T333 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 110169 T333 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 110169 T333 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 110182 T333 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 110188 T333 oass.SolrIndexSearcher.<init> Opening Searcher@2097b388 main
[junit4:junit4] 2> 110189 T333 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/tlog
[junit4:junit4] 2> 110190 T333 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 110190 T333 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 110195 T334 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2097b388 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 110197 T333 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 110198 T333 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 110275 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 110276 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:55117_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:55117/m_wd/c"}
[junit4:junit4] 2> 110277 T264 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 110277 T264 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 110289 T315 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 110289 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 110289 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 110289 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 110289 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 110289 T332 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> C15_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:60912_m_wd%2Fc, base_url=http://127.0.0.1:60912/m_wd/c}
[junit4:junit4] 2> 111076 T318 C15 P60912 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:20726/m_wd/c/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 111086 T318 C15 P60912 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&connTimeout=30000&socketTimeout=30000&retry=false
[junit4:junit4] 2> 111111 T318 C15 P60912 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:60912/m_wd/c START replicas=[http://127.0.0.1:20726/m_wd/c/collection1/] nUpdates=100
[junit4:junit4] 2> 111124 T318 C15 P60912 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 111125 T318 C15 P60912 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 111125 T318 C15 P60912 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 111125 T318 C15 P60912 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 111125 T318 C15 P60912 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 111126 T318 C15 P60912 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:20726/m_wd/c/collection1/. core=collection1
[junit4:junit4] 2> 111126 T318 C15 P60912 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C16 name=collection1 org.apache.solr.core.SolrCore@3e13bdf5 url=http://127.0.0.1:20726/m_wd/c/collection1 node=127.0.0.1:20726_m_wd%2Fc C16_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:20726_m_wd%2Fc, base_url=http://127.0.0.1:20726/m_wd/c, leader=true}
[junit4:junit4] 2> 111184 T274 C16 P20726 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1
[junit4:junit4] 2> 111207 T275 C16 P20726 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 111210 T275 C16 P20726 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4abacbab lockFactory=org.apache.lucene.store.NativeFSLockFactory@48e1dcac),segFN=segments_1,generation=1}
[junit4:junit4] 2> 111211 T275 C16 P20726 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 111212 T275 C16 P20726 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4abacbab lockFactory=org.apache.lucene.store.NativeFSLockFactory@48e1dcac),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4abacbab lockFactory=org.apache.lucene.store.NativeFSLockFactory@48e1dcac),segFN=segments_2,generation=2}
[junit4:junit4] 2> 111213 T275 C16 P20726 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 111214 T275 C16 P20726 oass.SolrIndexSearcher.<init> Opening Searcher@34fc91f realtime
[junit4:junit4] 2> 111214 T275 C16 P20726 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 111215 T275 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 32
[junit4:junit4] 2> 111217 T318 C15 P60912 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 111217 T318 C15 P60912 oascsi.HttpClientUtil.createClient Creating new http client, config:connTimeout=5000&socketTimeout=20000&allowCompression=false&maxConnections=10000&maxConnectionsPerHost=10000
[junit4:junit4] 2> 111226 T333 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 111226 T333 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55117/m_wd/c collection:collection1 shard:shard2
[junit4:junit4] 2> 111231 T333 oasc.ZkController.register We are http://127.0.0.1:55117/m_wd/c/collection1/ and leader is http://127.0.0.1:16736/m_wd/c/collection1/
[junit4:junit4] 2> 111232 T333 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55117/m_wd/c
[junit4:junit4] 2> 111232 T333 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 111232 T333 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C17 name=collection1 org.apache.solr.core.SolrCore@676068a9 url=http://127.0.0.1:55117/m_wd/c/collection1 node=127.0.0.1:55117_m_wd%2Fc C17_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:55117_m_wd%2Fc, base_url=http://127.0.0.1:55117/m_wd/c}
[junit4:junit4] 2> 111233 T336 C17 P55117 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 111234 T333 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 111234 T336 C17 P55117 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 111235 T336 C17 P55117 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 111235 T336 C17 P55117 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 111236 T241 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 111237 T241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 111237 T336 C17 P55117 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 111238 T241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 111245 T318 C15 P60912 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 111249 T276 C16 P20726 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 111250 T276 C16 P20726 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2
[junit4:junit4] 2> 111252 T318 C15 P60912 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 111252 T318 C15 P60912 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 111253 T318 C15 P60912 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 111256 T241 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 111256 T289 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 111257 T276 C16 P20726 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=1
[junit4:junit4] 2> 111258 T241 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 15 for each attempt
[junit4:junit4] 2> 111259 T318 C15 P60912 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 111259 T241 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4:junit4] 2> 111260 T318 C15 P60912 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/index.20130518192731319
[junit4:junit4] 2> 111261 T241 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 111261 T318 C15 P60912 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@1411aa8 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5d23192b) fullCopy=false
[junit4:junit4] 2> 111286 T276 C16 P20726 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=5
[junit4:junit4] 2> 111301 T318 C15 P60912 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 111303 T318 C15 P60912 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 111304 T318 C15 P60912 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 111311 T318 C15 P60912 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@f189a9f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6cf024e),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@f189a9f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6cf024e),segFN=segments_2,generation=2}
[junit4:junit4] 2> 111312 T318 C15 P60912 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 111313 T318 C15 P60912 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 111313 T318 C15 P60912 oass.SolrIndexSearcher.<init> Opening Searcher@38ae264a main
[junit4:junit4] 2> 111315 T317 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@38ae264a main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 111315 T318 C15 P60912 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/index.20130518192731319 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/index.20130518192731319;done=true>>]
[junit4:junit4] 2> 111316 T318 C15 P60912 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/index.20130518192731319
[junit4:junit4] 2> 111316 T318 C15 P60912 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty3/index.20130518192731319
[junit4:junit4] 2> 111317 T318 C15 P60912 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 111347 T318 C15 P60912 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 111347 T318 C15 P60912 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 111347 T318 C15 P60912 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 111350 T318 C15 P60912 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 111797 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 111799 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:55117_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:55117/m_wd/c"}
[junit4:junit4] 2> 111805 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:60912_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:60912/m_wd/c"}
[junit4:junit4] 2> 111819 T315 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 111819 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 111819 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 111819 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 111819 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 111819 T332 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 112258 T289 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 112258 T289 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:55117_m_wd%252Fc&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1002
[junit4:junit4] 2> 112263 T241 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 113267 T241 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> C17_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:55117_m_wd%2Fc, base_url=http://127.0.0.1:55117/m_wd/c}
[junit4:junit4] 2> 114260 T336 C17 P55117 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:16736/m_wd/c/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 114260 T336 C17 P55117 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:55117/m_wd/c START replicas=[http://127.0.0.1:16736/m_wd/c/collection1/] nUpdates=100
[junit4:junit4] 2> 114262 T336 C17 P55117 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 114262 T336 C17 P55117 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 114262 T336 C17 P55117 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 114263 T336 C17 P55117 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 114263 T336 C17 P55117 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 114263 T336 C17 P55117 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:16736/m_wd/c/collection1/. core=collection1
[junit4:junit4] 2> 114264 T336 C17 P55117 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 114269 T241 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> ASYNC NEW_CORE C18 name=collection1 org.apache.solr.core.SolrCore@75887693 url=http://127.0.0.1:16736/m_wd/c/collection1 node=127.0.0.1:16736_m_wd%2Fc C18_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:16736_m_wd%2Fc, base_url=http://127.0.0.1:16736/m_wd/c, leader=true}
[junit4:junit4] 2> 114273 T290 C18 P16736 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 114281 T291 C18 P16736 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 114283 T291 C18 P16736 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@fdc019b lockFactory=org.apache.lucene.store.NativeFSLockFactory@145a438c),segFN=segments_1,generation=1}
[junit4:junit4] 2> 114284 T291 C18 P16736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 114285 T291 C18 P16736 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@fdc019b lockFactory=org.apache.lucene.store.NativeFSLockFactory@145a438c),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@fdc019b lockFactory=org.apache.lucene.store.NativeFSLockFactory@145a438c),segFN=segments_2,generation=2}
[junit4:junit4] 2> 114285 T291 C18 P16736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 114286 T291 C18 P16736 oass.SolrIndexSearcher.<init> Opening Searcher@30b14998 realtime
[junit4:junit4] 2> 114286 T291 C18 P16736 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 114287 T291 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 7
[junit4:junit4] 2> 114288 T336 C17 P55117 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 114288 T336 C17 P55117 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 114291 T292 C18 P16736 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 114292 T292 C18 P16736 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=1
[junit4:junit4] 2> 114293 T336 C17 P55117 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 114293 T336 C17 P55117 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 114293 T336 C17 P55117 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 114298 T292 C18 P16736 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=1
[junit4:junit4] 2> 114299 T336 C17 P55117 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 114300 T336 C17 P55117 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index.20130518192734360
[junit4:junit4] 2> 114301 T336 C17 P55117 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@23be8694 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1db9b30c) fullCopy=false
[junit4:junit4] 2> 114314 T292 C18 P16736 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 114316 T336 C17 P55117 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 114318 T336 C17 P55117 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 114318 T336 C17 P55117 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 114320 T336 C17 P55117 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39900ea1 lockFactory=org.apache.lucene.store.NativeFSLockFactory@15863722),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39900ea1 lockFactory=org.apache.lucene.store.NativeFSLockFactory@15863722),segFN=segments_2,generation=2}
[junit4:junit4] 2> 114321 T336 C17 P55117 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 114321 T336 C17 P55117 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 114322 T336 C17 P55117 oass.SolrIndexSearcher.<init> Opening Searcher@7b010f3b main
[junit4:junit4] 2> 114323 T334 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b010f3b main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 114324 T336 C17 P55117 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index.20130518192734360 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index.20130518192734360;done=true>>]
[junit4:junit4] 2> 114324 T336 C17 P55117 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index.20130518192734360
[junit4:junit4] 2> 114324 T336 C17 P55117 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index.20130518192734360
[junit4:junit4] 2> 114324 T336 C17 P55117 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 114325 T336 C17 P55117 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 114325 T336 C17 P55117 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 114325 T336 C17 P55117 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 114329 T336 C17 P55117 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 114830 T264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 114832 T264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:55117_m_wd%2Fc",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:55117/m_wd/c"}
[junit4:junit4] 2> 114886 T315 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 114886 T270 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 114886 T283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 114886 T299 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 114886 T263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 114886 T332 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 115272 T241 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 115274 T241 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
[junit4:junit4] 2> ASYNC NEW_CORE C19 name=collection1 org.apache.solr.core.SolrCore@14ec56f url=http://127.0.0.1:15826/m_wd/c/collection1 node=127.0.0.1:15826_m_wd%2Fc C19_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:15826_m_wd%2Fc, base_url=http://127.0.0.1:15826/m_wd/c, leader=true}
[junit4:junit4] 2> 115285 T253 C19 P15826 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 115288 T253 C19 P15826 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5e8e11d5 lockFactory=org.apache.lucene.store.NativeFSLockFactory@11d75ba),segFN=segments_1,generation=1}
[junit4:junit4] 2> 115289 T253 C19 P15826 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 115290 T253 C19 P15826 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5e8e11d5 lockFactory=org.apache.lucene.store.NativeFSLockFactory@11d75ba),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5e8e11d5 lockFactory=org.apache.lucene.store.NativeFSLockFactory@11d75ba),segFN=segments_2,generation=2}
[junit4:junit4] 2> 115290 T253 C19 P15826 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 115291 T253 C19 P15826 oass.SolrIndexSearcher.<init> Opening Searcher@5c23c378 main
[junit4:junit4] 2> 115291 T253 C19 P15826 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 115292 T267 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5c23c378 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 115293 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 8
[junit4:junit4] 2> 115321 T293 C18 P16736 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 115322 T293 C18 P16736 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@fdc019b lockFactory=org.apache.lucene.store.NativeFSLockFactory@145a438c),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@fdc019b lockFactory=org.apache.lucene.store.NativeFSLockFactory@145a438c),segFN=segments_3,generation=3}
[junit4:junit4] 2> 115322 T293 C18 P16736 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 115323 T293 C18 P16736 oass.SolrIndexSearcher.<init> Opening Searcher@4999b280 main
[junit4:junit4] 2> 115324 T293 C18 P16736 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 115325 T301 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4999b280 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 115339 T293 C18 P16736 oasu.SolrCmdDistributor.distribCommit Distrib commit to:[StdNode: http://127.0.0.1:20726/m_wd/c/collection1/, StdNode: http://127.0.0.1:60912/m_wd/c/collection1/, StdNode: http://127.0.0.1:55117/m_wd/c/collection1/] params:commit_end_point=true&commit=true&softCommit=false&waitSearcher=true&expungeDeletes=false
[junit4:junit4] 2> 115348 T277 C16 P20726 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 115350 T277 C16 P20726 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4abacbab lockFactory=org.apache.lucene.store.NativeFSLockFactory@48e1dcac),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4abacbab lockFactory=org.apache.lucene.store.NativeFSLockFactory@48e1dcac),segFN=segments_3,generation=3}
[junit4:junit4] 2> 115350 T277 C16 P20726 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 115351 T277 C16 P20726 oass.SolrIndexSearcher.<init> Opening Searcher@65f9df0c main
[junit4:junit4] 2> 115351 T277 C16 P20726 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 115352 T285 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@65f9df0c main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 115353 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 5
[junit4:junit4] 2> C15_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:60912_m_wd%2Fc, base_url=http://127.0.0.1:60912/m_wd/c}
[junit4:junit4] 2> 115361 T305 C15 P60912 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> C17_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:55117_m_wd%2Fc, base_url=http://127.0.0.1:55117/m_wd/c}
[junit4:junit4] 2> 115361 T322 C17 P55117 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 115362 T305 C15 P60912 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@f189a9f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6cf024e),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@f189a9f lockFactory=org.apache.lucene.store.NativeFSLockFactory@6cf024e),segFN=segments_3,generation=3}
[junit4:junit4] 2> 115363 T305 C15 P60912 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 115363 T322 C17 P55117 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39900ea1 lockFactory=org.apache.lucene.store.NativeFSLockFactory@15863722),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@39900ea1 lockFactory=org.apache.lucene.store.NativeFSLockFactory@15863722),segFN=segments_3,generation=3}
[junit4:junit4] 2> 115364 T305 C15 P60912 oass.SolrIndexSearcher.<init> Opening Searcher@5ac6f0fd main
[junit4:junit4] 2> 115364 T322 C17 P55117 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 115365 T305 C15 P60912 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 115365 T322 C17 P55117 oass.SolrIndexSearcher.<init> Opening Searcher@54152dcd main
[junit4:junit4] 2> 115366 T317 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5ac6f0fd main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 115366 T322 C17 P55117 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 115367 T305 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 6
[junit4:junit4] 2> 115367 T334 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@54152dcd main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 115368 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 7
[junit4:junit4] 2> 115369 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 48
[junit4:junit4] 2> 115370 T241 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 115397 T278 C16 P20726 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1
[junit4:junit4] 2> 115401 T306 C15 P60912 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1
[junit4:junit4] 2> 115405 T294 C18 P16736 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1
[junit4:junit4] 2> 115408 T323 C17 P55117 oasc.SolrCore.execute [collection1] webapp=/m_wd/c path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 117480 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1435365337999605760)} 0 49
[junit4:junit4] 2> 117497 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={update.distrib=FROMLEADER&_version_=-1435365338035257344&update.from=http://127.0.0.1:16736/m_wd/c/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1435365338035257344)} 0 2
[junit4:junit4] 2> 117501 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={update.distrib=FROMLEADER&_version_=-1435365338046791680&update.from=http://127.0.0.1:20726/m_wd/c/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1435365338046791680)} 0 1
[junit4:junit4] 2> 117502 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={update.distrib=TOLEADER&wt=javabin&version=2} {deleteByQuery=*:* (-1435365338046791680)} 0 7
[junit4:junit4] 2> 117503 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1435365338035257344)} 0 19
[junit4:junit4] 2> 117515 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[0 (1435365338065666048)]} 0 7
[junit4:junit4] 2> 117542 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[0 (1435365338089783296)]} 0 1
[junit4:junit4] 2> 117544 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[0 (1435365338089783296)]} 0 8
[junit4:junit4] 2> 117544 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[0]} 0 25
[junit4:junit4] 2> 117549 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[1 (1435365338103414784)]} 0 0
[junit4:junit4] 2> 117562 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[1 (1435365338111803392)]} 0 1
[junit4:junit4] 2> 117563 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[1 (1435365338111803392)]} 0 6
[junit4:junit4] 2> 117563 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[1]} 0 9
[junit4:junit4] 2> 117567 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[2 (1435365338122289152)]} 0 1
[junit4:junit4] 2> 117579 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[2 (1435365338129629184)]} 0 1
[junit4:junit4] 2> 117581 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[2 (1435365338129629184)]} 0 7
[junit4:junit4] 2> 117582 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[2]} 0 11
[junit4:junit4] 2> 117586 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[3 (1435365338142212096)]} 0 0
[junit4:junit4] 2> 117596 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[3 (1435365338147454976)]} 0 1
[junit4:junit4] 2> 117597 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[3 (1435365338147454976)]} 0 6
[junit4:junit4] 2> 117601 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[4 (1435365338157940736)]} 0 0
[junit4:junit4] 2> 117615 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[4 (1435365338167377920)]} 0 1
[junit4:junit4] 2> 117616 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[4 (1435365338167377920)]} 0 6
[junit4:junit4] 2> 117617 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[4]} 0 11
[junit4:junit4] 2> 117621 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[5 (1435365338178912256)]} 0 0
[junit4:junit4] 2> 117634 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[5 (1435365338187300864)]} 0 1
[junit4:junit4] 2> 117635 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[5 (1435365338187300864)]} 0 6
[junit4:junit4] 2> 117636 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[5]} 0 10
[junit4:junit4] 2> 117640 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[6 (1435365338198835200)]} 0 0
[junit4:junit4] 2> 117650 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[6 (1435365338204078080)]} 0 1
[junit4:junit4] 2> 117651 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[6 (1435365338204078080)]} 0 6
[junit4:junit4] 2> 117655 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[7 (1435365338214563840)]} 0 0
[junit4:junit4] 2> 117669 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[7 (1435365338222952448)]} 0 1
[junit4:junit4] 2> 117670 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[7 (1435365338222952448)]} 0 7
[junit4:junit4] 2> 117671 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[7]} 0 11
[junit4:junit4] 2> 117675 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[8 (1435365338235535360)]} 0 0
[junit4:junit4] 2> 117687 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[8 (1435365338242875392)]} 0 1
[junit4:junit4] 2> 117688 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[8 (1435365338242875392)]} 0 6
[junit4:junit4] 2> 117689 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[8]} 0 10
[junit4:junit4] 2> 117694 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[9 (1435365338254409728)]} 0 1
[junit4:junit4] 2> 117703 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[9 (1435365338259652608)]} 0 1
[junit4:junit4] 2> 117704 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[9 (1435365338259652608)]} 0 6
[junit4:junit4] 2> 117709 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[10 (1435365338270138368)]} 0 1
[junit4:junit4] 2> 117721 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[10 (1435365338278526976)]} 0 1
[junit4:junit4] 2> 117721 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[10 (1435365338278526976)]} 0 5
[junit4:junit4] 2> 117722 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[10]} 0 9
[junit4:junit4] 2> 117728 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[11 (1435365338290061312)]} 0 1
[junit4:junit4] 2> 117736 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[11 (1435365338295304192)]} 0 0
[junit4:junit4] 2> 117737 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[11 (1435365338295304192)]} 0 5
[junit4:junit4] 2> 117742 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[12 (1435365338304741376)]} 0 1
[junit4:junit4] 2> 117754 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[12 (1435365338313129984)]} 0 1
[junit4:junit4] 2> 117755 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[12 (1435365338313129984)]} 0 6
[junit4:junit4] 2> 117756 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[12]} 0 10
[junit4:junit4] 2> 117761 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[13 (1435365338324664320)]} 0 1
[junit4:junit4] 2> 117774 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[13 (1435365338334101504)]} 0 1
[junit4:junit4] 2> 117775 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[13 (1435365338334101504)]} 0 6
[junit4:junit4] 2> 117776 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[13]} 0 11
[junit4:junit4] 2> 117781 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[14 (1435365338345635840)]} 0 1
[junit4:junit4] 2> 117790 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[14 (1435365338350878720)]} 0 1
[junit4:junit4] 2> 117791 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[14 (1435365338350878720)]} 0 6
[junit4:junit4] 2> 117796 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[15 (1435365338361364480)]} 0 1
[junit4:junit4] 2> 117805 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[15 (1435365338366607360)]} 0 1
[junit4:junit4] 2> 117805 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[15 (1435365338366607360)]} 0 6
[junit4:junit4] 2> 117810 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[16 (1435365338377093120)]} 0 0
[junit4:junit4] 2> 117824 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[16 (1435365338386530304)]} 0 1
[junit4:junit4] 2> 117825 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[16 (1435365338386530304)]} 0 6
[junit4:junit4] 2> 117825 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[16]} 0 10
[junit4:junit4] 2> 117829 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[17 (1435365338397016064)]} 0 0
[junit4:junit4] 2> 117842 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[17 (1435365338405404672)]} 0 1
[junit4:junit4] 2> 117843 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[17 (1435365338405404672)]} 0 6
[junit4:junit4] 2> 117844 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[17]} 0 11
[junit4:junit4] 2> 117848 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[18 (1435365338416939008)]} 0 0
[junit4:junit4] 2> 117858 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[18 (1435365338422181888)]} 0 1
[junit4:junit4] 2> 117859 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[18 (1435365338422181888)]} 0 6
[junit4:junit4] 2> 117864 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[19 (1435365338432667648)]} 0 1
[junit4:junit4] 2> 117873 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[19 (1435365338437910528)]} 0 1
[junit4:junit4] 2> 117874 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[19 (1435365338437910528)]} 0 7
[junit4:junit4] 2> 117878 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[20 (1435365338448396288)]} 0 0
[junit4:junit4] 2> 117891 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[20 (1435365338456784896)]} 0 1
[junit4:junit4] 2> 117892 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[20 (1435365338456784896)]} 0 6
[junit4:junit4] 2> 117893 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[20]} 0 10
[junit4:junit4] 2> 117897 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[21 (1435365338468319232)]} 0 0
[junit4:junit4] 2> 117907 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[21 (1435365338473562112)]} 0 1
[junit4:junit4] 2> 117908 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[21 (1435365338473562112)]} 0 6
[junit4:junit4] 2> 117912 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[22 (1435365338484047872)]} 0 0
[junit4:junit4] 2> 117924 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[22 (1435365338491387904)]} 0 1
[junit4:junit4] 2> 117925 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[22 (1435365338491387904)]} 0 6
[junit4:junit4] 2> 117925 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[22]} 0 9
[junit4:junit4] 2> 117930 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[23 (1435365338502922240)]} 0 1
[junit4:junit4] 2> 117939 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[23 (1435365338507116544)]} 0 1
[junit4:junit4] 2> 117940 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[23 (1435365338507116544)]} 0 6
[junit4:junit4] 2> 117945 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[24 (1435365338517602304)]} 0 1
[junit4:junit4] 2> 117958 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[24 (1435365338527039488)]} 0 1
[junit4:junit4] 2> 117958 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[24 (1435365338527039488)]} 0 5
[junit4:junit4] 2> 117959 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[24]} 0 10
[junit4:junit4] 2> 117964 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[25 (1435365338537525248)]} 0 1
[junit4:junit4] 2> 117976 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[25 (1435365338545913856)]} 0 1
[junit4:junit4] 2> 117977 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[25 (1435365338545913856)]} 0 6
[junit4:junit4] 2> 117978 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[25]} 0 10
[junit4:junit4] 2> 117982 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[26 (1435365338557448192)]} 0 0
[junit4:junit4] 2> 117992 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[26 (1435365338562691072)]} 0 1
[junit4:junit4] 2> 117993 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[26 (1435365338562691072)]} 0 6
[junit4:junit4] 2> 117998 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[27 (1435365338573176832)]} 0 1
[junit4:junit4] 2> 118006 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[27 (1435365338578419712)]} 0 0
[junit4:junit4] 2> 118007 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[27 (1435365338578419712)]} 0 5
[junit4:junit4] 2> 118012 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[28 (1435365338588905472)]} 0 0
[junit4:junit4] 2> 118022 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[28 (1435365338594148352)]} 0 1
[junit4:junit4] 2> 118023 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[28 (1435365338594148352)]} 0 7
[junit4:junit4] 2> 118027 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[29 (1435365338604634112)]} 0 1
[junit4:junit4] 2> 118036 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[29 (1435365338608828416)]} 0 1
[junit4:junit4] 2> 118037 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[29 (1435365338608828416)]} 0 6
[junit4:junit4] 2> 118041 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[30 (1435365338619314176)]} 0 0
[junit4:junit4] 2> 118054 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[30 (1435365338627702784)]} 0 1
[junit4:junit4] 2> 118055 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[30 (1435365338627702784)]} 0 6
[junit4:junit4] 2> 118056 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[30]} 0 10
[junit4:junit4] 2> 118060 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[31 (1435365338639237120)]} 0 1
[junit4:junit4] 2> 118072 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[31 (1435365338646577152)]} 0 1
[junit4:junit4] 2> 118073 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[31 (1435365338646577152)]} 0 6
[junit4:junit4] 2> 118074 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[31]} 0 10
[junit4:junit4] 2> 118079 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[32 (1435365338658111488)]} 0 1
[junit4:junit4] 2> 118090 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[32 (1435365338666500096)]} 0 0
[junit4:junit4] 2> 118091 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[32 (1435365338666500096)]} 0 5
[junit4:junit4] 2> 118092 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[32]} 0 9
[junit4:junit4] 2> 118096 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[33 (1435365338676985856)]} 0 1
[junit4:junit4] 2> 118109 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[33 (1435365338685374464)]} 0 1
[junit4:junit4] 2> 118110 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[33 (1435365338685374464)]} 0 6
[junit4:junit4] 2> 118111 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[33]} 0 11
[junit4:junit4] 2> 118116 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[34 (1435365338697957376)]} 0 1
[junit4:junit4] 2> 118127 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[34 (1435365338705297408)]} 0 0
[junit4:junit4] 2> 118128 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[34 (1435365338705297408)]} 0 5
[junit4:junit4] 2> 118129 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[34]} 0 9
[junit4:junit4] 2> 118134 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[35 (1435365338716831744)]} 0 1
[junit4:junit4] 2> 118145 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[35 (1435365338724171776)]} 0 0
[junit4:junit4] 2> 118146 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[35 (1435365338724171776)]} 0 5
[junit4:junit4] 2> 118147 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[35]} 0 9
[junit4:junit4] 2> 118152 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[36 (1435365338735706112)]} 0 0
[junit4:junit4] 2> 118163 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[36 (1435365338743046144)]} 0 0
[junit4:junit4] 2> 118164 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[36 (1435365338743046144)]} 0 5
[junit4:junit4] 2> 118165 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[36]} 0 10
[junit4:junit4] 2> 118169 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[37 (1435365338753531904)]} 0 0
[junit4:junit4] 2> 118181 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[37 (1435365338761920512)]} 0 0
[junit4:junit4] 2> 118183 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[37 (1435365338761920512)]} 0 6
[junit4:junit4] 2> 118184 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[37]} 0 11
[junit4:junit4] 2> 118188 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[38 (1435365338773454848)]} 0 1
[junit4:junit4] 2> 118196 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[38 (1435365338777649152)]} 0 0
[junit4:junit4] 2> 118197 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[38 (1435365338777649152)]} 0 5
[junit4:junit4] 2> 118201 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[39 (1435365338787086336)]} 0 1
[junit4:junit4] 2> 118210 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[39 (1435365338791280640)]} 0 1
[junit4:junit4] 2> 118211 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[39 (1435365338791280640)]} 0 7
[junit4:junit4] 2> 118215 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[40 (1435365338801766400)]} 0 1
[junit4:junit4] 2> 118223 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[40 (1435365338805960704)]} 0 0
[junit4:junit4] 2> 118224 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[40 (1435365338805960704)]} 0 5
[junit4:junit4] 2> 118228 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[41 (1435365338815397888)]} 0 1
[junit4:junit4] 2> 118236 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[41 (1435365338819592192)]} 0 0
[junit4:junit4] 2> 118237 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[41 (1435365338819592192)]} 0 5
[junit4:junit4] 2> 118241 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[42 (1435365338829029376)]} 0 0
[junit4:junit4] 2> 118262 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[42 (1435365338845806592)]} 0 1
[junit4:junit4] 2> 118263 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[42 (1435365338845806592)]} 0 7
[junit4:junit4] 2> 118264 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[42]} 0 19
[junit4:junit4] 2> 118269 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[43 (1435365338857340928)]} 0 1
[junit4:junit4] 2> 118281 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[43 (1435365338865729536)]} 0 1
[junit4:junit4] 2> 118282 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[43 (1435365338865729536)]} 0 6
[junit4:junit4] 2> 118283 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[43]} 0 11
[junit4:junit4] 2> 118287 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[44 (1435365338877263872)]} 0 0
[junit4:junit4] 2> 118297 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[44 (1435365338882506752)]} 0 1
[junit4:junit4] 2> 118298 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[44 (1435365338882506752)]} 0 6
[junit4:junit4] 2> 118302 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[45 (1435365338892992512)]} 0 0
[junit4:junit4] 2> 118311 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[45 (1435365338898235392)]} 0 1
[junit4:junit4] 2> 118313 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[45 (1435365338898235392)]} 0 6
[junit4:junit4] 2> 118317 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[46 (1435365338908721152)]} 0 1
[junit4:junit4] 2> 118326 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[46 (1435365338912915456)]} 0 1
[junit4:junit4] 2> 118327 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[46 (1435365338912915456)]} 0 6
[junit4:junit4] 2> 118331 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[47 (1435365338923401216)]} 0 1
[junit4:junit4] 2> 118340 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[47 (1435365338927595520)]} 0 1
[junit4:junit4] 2> 118341 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[47 (1435365338927595520)]} 0 6
[junit4:junit4] 2> 118345 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[48 (1435365338938081280)]} 0 0
[junit4:junit4] 2> 118354 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[48 (1435365338943324160)]} 0 1
[junit4:junit4] 2> 118356 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[48 (1435365338943324160)]} 0 6
[junit4:junit4] 2> 118360 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[49 (1435365338953809920)]} 0 1
[junit4:junit4] 2> 118370 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[49 (1435365338959052800)]} 0 1
[junit4:junit4] 2> 118371 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[49 (1435365338959052800)]} 0 7
[junit4:junit4] 2> 118375 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[50 (1435365338969538560)]} 0 1
[junit4:junit4] 2> 118384 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[50 (1435365338973732864)]} 0 1
[junit4:junit4] 2> 118385 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[50 (1435365338973732864)]} 0 7
[junit4:junit4] 2> 118389 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[51 (1435365338984218624)]} 0 1
[junit4:junit4] 2> 118397 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[51 (1435365338988412928)]} 0 1
[junit4:junit4] 2> 118399 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[51 (1435365338988412928)]} 0 7
[junit4:junit4] 2> 118404 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[52 (1435365338998898688)]} 0 2
[junit4:junit4] 2> 118416 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[52 (1435365339007287296)]} 0 1
[junit4:junit4] 2> 118417 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[52 (1435365339007287296)]} 0 7
[junit4:junit4] 2> 118418 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[52]} 0 11
[junit4:junit4] 2> 118423 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[53 (1435365339018821632)]} 0 1
[junit4:junit4] 2> 118435 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[53 (1435365339028258816)]} 0 0
[junit4:junit4] 2> 118436 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[53 (1435365339028258816)]} 0 5
[junit4:junit4] 2> 118437 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[53]} 0 10
[junit4:junit4] 2> 118442 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[54 (1435365339038744576)]} 0 1
[junit4:junit4] 2> 118452 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[54 (1435365339047133184)]} 0 1
[junit4:junit4] 2> 118453 T277 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[54 (1435365339047133184)]} 0 5
[junit4:junit4] 2> 118454 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[54]} 0 8
[junit4:junit4] 2> 118457 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[55 (1435365339055521792)]} 0 0
[junit4:junit4] 2> 118464 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[55 (1435365339059716096)]} 0 1
[junit4:junit4] 2> 118465 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[55 (1435365339059716096)]} 0 5
[junit4:junit4] 2> 118468 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[56 (1435365339066007552)]} 0 1
[junit4:junit4] 2> 118479 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[56 (1435365339074396160)]} 0 1
[junit4:junit4] 2> 118480 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[56 (1435365339074396160)]} 0 6
[junit4:junit4] 2> 118481 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[56]} 0 10
[junit4:junit4] 2> 118485 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[57 (1435365339084881920)]} 0 1
[junit4:junit4] 2> 118495 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[57 (1435365339092221952)]} 0 1
[junit4:junit4] 2> 118496 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[57 (1435365339092221952)]} 0 5
[junit4:junit4] 2> 118497 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[57]} 0 9
[junit4:junit4] 2> 118501 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[58 (1435365339101659136)]} 0 1
[junit4:junit4] 2> 118509 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[58 (1435365339105853440)]} 0 1
[junit4:junit4] 2> 118510 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[58 (1435365339105853440)]} 0 6
[junit4:junit4] 2> 118514 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[59 (1435365339114242048)]} 0 1
[junit4:junit4] 2> 118521 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[59 (1435365339119484928)]} 0 1
[junit4:junit4] 2> 118522 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[59 (1435365339119484928)]} 0 5
[junit4:junit4] 2> 118526 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[60 (1435365339127873536)]} 0 1
[junit4:junit4] 2> 118536 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[60 (1435365339135213568)]} 0 1
[junit4:junit4] 2> 118537 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[60 (1435365339135213568)]} 0 5
[junit4:junit4] 2> 118538 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[60]} 0 9
[junit4:junit4] 2> 118542 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[61 (1435365339144650752)]} 0 1
[junit4:junit4] 2> 118549 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[61 (1435365339148845056)]} 0 1
[junit4:junit4] 2> 118550 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[61 (1435365339148845056)]} 0 5
[junit4:junit4] 2> 118554 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[62 (1435365339157233664)]} 0 1
[junit4:junit4] 2> 118564 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[62 (1435365339164573696)]} 0 1
[junit4:junit4] 2> 118565 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[62 (1435365339164573696)]} 0 5
[junit4:junit4] 2> 118566 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[62]} 0 9
[junit4:junit4] 2> 118570 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[63 (1435365339174010880)]} 0 1
[junit4:junit4] 2> 118577 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[63 (1435365339178205184)]} 0 1
[junit4:junit4] 2> 118578 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[63 (1435365339178205184)]} 0 5
[junit4:junit4] 2> 118582 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[64 (1435365339186593792)]} 0 1
[junit4:junit4] 2> 118589 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[64 (1435365339190788096)]} 0 1
[junit4:junit4] 2> 118590 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[64 (1435365339190788096)]} 0 5
[junit4:junit4] 2> 118593 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[65 (1435365339198128128)]} 0 1
[junit4:junit4] 2> 118600 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[65 (1435365339201273856)]} 0 1
[junit4:junit4] 2> 118601 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[65 (1435365339201273856)]} 0 5
[junit4:junit4] 2> 118604 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[66 (1435365339209662464)]} 0 1
[junit4:junit4] 2> 118611 T307 C15 P60912 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[66 (1435365339213856768)]} 0 1
[junit4:junit4] 2> 118611 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[66 (1435365339213856768)]} 0 4
[junit4:junit4] 2> 118615 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[67 (1435365339221196800)]} 0 1
[junit4:junit4] 2> 118625 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[67 (1435365339228536832)]} 0 1
[junit4:junit4] 2> 118626 T295 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:20726/m_wd/c/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[67 (1435365339228536832)]} 0 5
[junit4:junit4] 2> 118627 T279 C16 P20726 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[67]} 0 9
[junit4:junit4] 2> 118630 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[68 (1435365339236925440)]} 0 1
[junit4:junit4] 2> 118637 T322 C17 P55117 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={distrib.from=http://127.0.0.1:16736/m_wd/c/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[68 (1435365339241119744)]} 0 1
[junit4:junit4] 2> 118638 T293 C18 P16736 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update params={wt=javabin&version=2} {add=[68 (1435365339241119744)]} 0 5
[junit4:junit4] 2> 118642 T253 C19 P15826 oasup.LogUpdateProcessor.finish [collection1] webapp=/m_wd/c path=/update
[...truncated too long message...]
tem property
[junit4:junit4] 2> 349790 T241 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@676068a9
[junit4:junit4] 2> 349792 T241 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=737,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
[junit4:junit4] 2> 349793 T241 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
[junit4:junit4] 2> 349793 T241 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
[junit4:junit4] 2> 349794 T241 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
[junit4:junit4] 2> 349796 T241 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
[junit4:junit4] 2> 349797 T241 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
[junit4:junit4] 2> 349797 T241 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4;done=false>>]
[junit4:junit4] 2> 349798 T241 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4
[junit4:junit4] 2> 349798 T241 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index;done=false>>]
[junit4:junit4] 2> 349799 T241 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133/jetty4/index
[junit4:junit4] 2> 349800 T429 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=89710331896659980-127.0.0.1:55117_m_wd%2Fc-n_0000000004) am no longer a leader.
[junit4:junit4] 2> 349801 T422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 349802 T241 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/m_wd/c,null}
[junit4:junit4] 2> 349803 T422 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4:junit4] 2> 349875 T241 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 349877 T241 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:15823 15823
[junit4:junit4] 2> 350110 T241 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 15826
[junit4:junit4] 2> !!!! WARNING: best effort to remove /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368871031133 FAILED !!!!!
[junit4:junit4] 2> 350111 T241 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
[junit4:junit4] 2> 350112 T241 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:15823 15823
[junit4:junit4] 1> "core":"collection1_shard1_0_replica1",
[junit4:junit4] 1> "node_name":"127.0.0.1:60912_m_wd%2Fc",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:60912/m_wd/c"}
[junit4:junit4] 1> /solr/collections/collection1/leaders/shard1_1 (0)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {
[junit4:junit4] 1> "core":"collection1_shard1_1_replica1",
[junit4:junit4] 1> "node_name":"127.0.0.1:60912_m_wd%2Fc",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:60912/m_wd/c"}
[junit4:junit4] 1> /solr/collections/collection1/leaders/shard2 (0)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:16736_m_wd%2Fc",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:16736/m_wd/c"}
[junit4:junit4] 1> /solr/collections/control_collection (3)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {"configName":"conf1"}
[junit4:junit4] 1> /solr/collections/control_collection/shards (0)
[junit4:junit4] 1> /solr/collections/control_collection/leader_elect (1)
[junit4:junit4] 1> /solr/collections/control_collection/leader_elect/shard1 (1)
[junit4:junit4] 1> /solr/collections/control_collection/leader_elect/shard1/election (1)
[junit4:junit4] 1> /solr/collections/control_collection/leader_elect/shard1/election/89710331896659971-1-n_0000000000 (0)
[junit4:junit4] 1> /solr/collections/control_collection/leaders (1)
[junit4:junit4] 1> /solr/collections/control_collection/leaders/shard1 (0)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:15826_m_wd%2Fc",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:15826/m_wd/c"}
[junit4:junit4] 1> /solr/overseer_elect (2)
[junit4:junit4] 1> /solr/overseer_elect/election (6)
[junit4:junit4] 1> /solr/overseer_elect/election/89710331896659984-127.0.0.1:15823_solr-n_0000000007 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89710331896659978-127.0.0.1:60912_m_wd%2Fc-n_0000000003 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89710331896659971-127.0.0.1:15826_m_wd%2Fc-n_0000000000 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89710331896659982-127.0.0.1:20726_m_wd%2Fc-n_0000000005 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89710331896659980-127.0.0.1:55117_m_wd%2Fc-n_0000000004 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89710331896659976-127.0.0.1:16736_m_wd%2Fc-n_0000000002 (0)
[junit4:junit4] 1> /solr/overseer_elect/leader (0)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {"id":"89710331896659971-127.0.0.1:15826_m_wd%2Fc-n_0000000000"}
[junit4:junit4] 1> /zookeeper (1)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1>
[junit4:junit4] 1>
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeyShardSplitTest -Dtests.method=testDistribSearch -Dtests.seed=74BCBF895EB06A15 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=en_US -Dtests.timezone=Australia/Darwin -Dtests.file.encoding=US-ASCII
[junit4:junit4] FAILURE 259s J0 | ChaosMonkeyShardSplitTest.testDistribSearch <<<
[junit4:junit4] > Throwable #1: java.lang.AssertionError: Wrong doc count on shard1_1 expected:<400> but was:<399>
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([74BCBF895EB06A15:F55A319129EF0A29]:0)
[junit4:junit4] > at org.apache.solr.cloud.ShardSplitTest.checkDocCountsAndShardStates(ShardSplitTest.java:167)
[junit4:junit4] > at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:155)
[junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4:junit4] 2> 350265 T241 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4] 2> 259203 T240 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
[junit4:junit4] 2> 351102 T332 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 351102 T332 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 351102 T332 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> NOTE: test params are: codec=Lucene40, sim=DefaultSimilarity, locale=en_US, timezone=Australia/Darwin
[junit4:junit4] 2> NOTE: FreeBSD 9.0-RELEASE amd64/Oracle Corporation 1.7.0_17 (64-bit)/cpus=16,threads=2,free=57165024,total=119537664
[junit4:junit4] 2> NOTE: All tests run in this JVM: [TestZkChroot, TestCoreContainer, TestExtendedDismaxParser, TestSerializedLuceneMatchVersion, ExternalFileFieldSortTest, BadCopyFieldTest, TestMultiCoreConfBootstrap, DirectUpdateHandlerTest, NumericFieldsTest, TestManagedSchemaFieldResource, WordBreakSolrSpellCheckerTest, TestRangeQuery, TestRTGBase, XsltUpdateRequestHandlerTest, ChaosMonkeyShardSplitTest]
[junit4:junit4] Completed on J0 in 260.23s, 1 test, 1 failure <<< FAILURES!
[...truncated 773 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/build.xml:383: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/build.xml:363: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build.xml:181: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/common-build.xml:437: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/lucene/common-build.xml:1243: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/lucene/common-build.xml:887: There were test failures: 296 suites, 1234 tests, 1 failure, 27 ignored (8 assumptions)
Total time: 48 minutes 20 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1263 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1263/
No tests ran.
Build Log:
[...truncated 7003 lines...]
AU FATAL: hudson.remoting.RequestAbortedException: java.io.IOException: Unexpected termination of the channel
hudson.remoting.RequestAbortedException: hudson.remoting.RequestAbortedException: java.io.IOException: Unexpected termination of the channel
at hudson.remoting.Request.call(Request.java:174)
at hudson.remoting.Channel.call(Channel.java:672)
at hudson.FilePath.act(FilePath.java:854)
at hudson.FilePath.act(FilePath.java:838)
at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:743)
at hudson.scm.SubversionSCM.checkout(SubversionSCM.java:685)
at hudson.model.AbstractProject.checkout(AbstractProject.java:1331)
at hudson.model.AbstractBuild$AbstractBuildExecution.defaultCheckout(AbstractBuild.java:682)
at jenkins.scm.SCMCheckoutStrategy.checkout(SCMCheckoutStrategy.java:88)
at hudson.model.AbstractBuild$AbstractBuildExecution.run(AbstractBuild.java:587)
at hudson.model.Run.execute(Run.java:1568)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:46)
at hudson.model.ResourceController.execute(ResourceController.java:88)
at hudson.model.Executor.run(Executor.java:236)
Caused by: hudson.remoting.RequestAbortedException: java.io.IOException: Unexpected termination of the channel
at hudson.remoting.Request.abort(Request.java:299)
at hudson.remoting.Channel.terminate(Channel.java:732)
at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:69)
Caused by: java.io.IOException: Unexpected termination of the channel
at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:50)
Caused by: java.io.EOFException
at java.io.ObjectInputStream$BlockDataInputStream.peekByte(ObjectInputStream.java:2577)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1315)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369)
at hudson.remoting.Command.readFrom(Command.java:92)
at hudson.remoting.ClassicCommandTransport.read(ClassicCommandTransport.java:59)
at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:48)
[JENKINS] Lucene-Solr-Tests-4.x-java7 - Build # 1262 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-java7/1262/
1 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeyShardSplitTest.testDistribSearch
Error Message:
Server at http://127.0.0.1:43350/zx/cp returned non ok status:500, message:Server Error
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:43350/zx/cp returned non ok status:500, message:Server Error
at __randomizedtesting.SeedInfo.seed([B293A12E2D6924FD:33752F365A3644C1]:0)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:208)
at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:136)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:722)
Build Log:
[...truncated 9344 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeyShardSplitTest
[junit4:junit4] 2> 80306 T394 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /zx/cp
[junit4:junit4] 2> 80314 T394 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./solrtest-ChaosMonkeyShardSplitTest-1368952133840
[junit4:junit4] 2> 80316 T394 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 80316 T395 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 80417 T394 oasc.ZkTestServer.run start zk server on port:43343
[junit4:junit4] 2> 80418 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 80425 T401 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@43700de8 name:ZooKeeperConnection Watcher:127.0.0.1:43343 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 80426 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 80426 T394 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 80432 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 80443 T403 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3db9bdbb name:ZooKeeperConnection Watcher:127.0.0.1:43343/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 80444 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 80444 T394 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 80450 T394 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 80455 T394 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 80460 T394 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 80465 T394 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 80466 T394 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 80482 T394 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 80483 T394 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 80614 T394 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 80615 T394 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 80626 T394 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 80627 T394 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 80633 T394 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 80634 T394 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 80639 T394 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 80640 T394 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 80645 T394 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 80646 T394 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 80651 T394 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 80652 T394 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 80657 T394 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 80658 T394 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 80889 T394 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 80894 T394 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43346
[junit4:junit4] 2> 80895 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 80896 T394 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 80896 T394 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368952134190
[junit4:junit4] 2> 80897 T394 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368952134190/solr.xml
[junit4:junit4] 2> 80898 T394 oasc.CoreContainer.<init> New CoreContainer 2061703705
[junit4:junit4] 2> 80899 T394 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368952134190/'
[junit4:junit4] 2> 80899 T394 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368952134190/'
[junit4:junit4] 2> 81019 T394 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 81020 T394 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 81021 T394 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 81022 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 81022 T394 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 81023 T394 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 81023 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 81024 T394 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 81024 T394 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 81025 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 81043 T394 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 81044 T394 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43343/solr
[junit4:junit4] 2> 81063 T394 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 81064 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 81067 T414 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d45094f name:ZooKeeperConnection Watcher:127.0.0.1:43343 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 81068 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 81075 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 81094 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 81096 T416 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64920fca name:ZooKeeperConnection Watcher:127.0.0.1:43343/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 81096 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 81099 T394 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 81106 T394 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 81111 T394 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 81115 T394 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43346_zx%2Fcp
[junit4:junit4] 2> 81117 T394 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43346_zx%2Fcp
[junit4:junit4] 2> 81127 T394 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 81143 T394 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 81148 T394 oasc.Overseer.start Overseer (id=89715647043731459-127.0.0.1:43346_zx%2Fcp-n_0000000000) starting
[junit4:junit4] 2> 81168 T394 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 81186 T418 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 81187 T394 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 81191 T394 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 81195 T394 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 81220 T417 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 81232 T419 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368952134190/collection1
[junit4:junit4] 2> 81233 T419 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 81234 T419 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 81234 T419 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 81237 T419 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368952134190/collection1/'
[junit4:junit4] 2> 81238 T419 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368952134190/collection1/lib/README' to classloader
[junit4:junit4] 2> 81239 T419 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368952134190/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 81320 T419 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 81384 T419 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 81486 T419 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 81493 T419 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 82107 T419 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 82111 T419 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 82114 T419 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 82120 T419 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 82166 T419 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 82166 T419 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368952134190/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/control/data/
[junit4:junit4] 2> 82167 T419 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d70b8c3
[junit4:junit4] 2> 82167 T419 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 82168 T419 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/control/data
[junit4:junit4] 2> 82169 T419 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/control/data/index/
[junit4:junit4] 2> 82169 T419 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 82170 T419 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/control/data/index
[junit4:junit4] 2> 82174 T419 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@ac49d10 lockFactory=org.apache.lucene.store.NativeFSLockFactory@8c2cbff),segFN=segments_1,generation=1}
[junit4:junit4] 2> 82175 T419 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 82178 T419 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 82178 T419 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 82179 T419 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 82180 T419 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 82180 T419 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 82181 T419 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 82181 T419 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 82182 T419 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 82182 T419 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 82196 T419 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 82202 T419 oass.SolrIndexSearcher.<init> Opening Searcher@59d08ee4 main
[junit4:junit4] 2> 82203 T419 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/control/data/tlog
[junit4:junit4] 2> 82204 T419 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 82204 T419 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 82209 T420 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@59d08ee4 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 82212 T419 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 82212 T419 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 82730 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 82732 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43346_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43346/zx/cp"}
[junit4:junit4] 2> 82733 T417 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 82760 T417 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 82779 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 83216 T419 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 83216 T419 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43346/zx/cp collection:control_collection shard:shard1
[junit4:junit4] 2> 83230 T419 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
[junit4:junit4] 2> 83250 T419 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 83272 T419 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 83276 T419 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 83277 T419 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 83277 T419 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:43346/zx/cp/collection1/
[junit4:junit4] 2> 83278 T419 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 83278 T419 oasc.SyncStrategy.syncToMe http://127.0.0.1:43346/zx/cp/collection1/ has no replicas
[junit4:junit4] 2> 83278 T419 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:43346/zx/cp/collection1/
[junit4:junit4] 2> 83279 T419 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 84287 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 84313 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 84361 T419 oasc.ZkController.register We are http://127.0.0.1:43346/zx/cp/collection1/ and leader is http://127.0.0.1:43346/zx/cp/collection1/
[junit4:junit4] 2> 84361 T419 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43346/zx/cp
[junit4:junit4] 2> 84361 T419 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 84362 T419 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 84362 T419 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 84365 T419 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 84367 T394 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 84367 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 84368 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 84381 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 84387 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 84390 T423 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a3ef897 name:ZooKeeperConnection Watcher:127.0.0.1:43343/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 84390 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 84392 T394 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 84398 T394 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 84619 T394 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 84623 T394 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43350
[junit4:junit4] 2> 84624 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 84624 T394 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 84625 T394 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368952137924
[junit4:junit4] 2> 84626 T394 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368952137924/solr.xml
[junit4:junit4] 2> 84626 T394 oasc.CoreContainer.<init> New CoreContainer 165160801
[junit4:junit4] 2> 84627 T394 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368952137924/'
[junit4:junit4] 2> 84628 T394 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368952137924/'
[junit4:junit4] 2> 84745 T394 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 84746 T394 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 84747 T394 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 84747 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 84748 T394 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 84748 T394 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 84749 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 84750 T394 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 84750 T394 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 84751 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 84767 T394 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 84768 T394 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43343/solr
[junit4:junit4] 2> 84768 T394 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 84770 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 84773 T434 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fdb47b9 name:ZooKeeperConnection Watcher:127.0.0.1:43343 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 84774 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 84788 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 84801 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 84803 T436 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@54714b43 name:ZooKeeperConnection Watcher:127.0.0.1:43343/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 84803 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 84816 T394 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 85822 T394 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43350_zx%2Fcp
[junit4:junit4] 2> 85824 T394 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43350_zx%2Fcp
[junit4:junit4] 2> 85828 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 85830 T423 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 85830 T436 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 85831 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 85832 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43346_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43346/zx/cp"}
[junit4:junit4] 2> 85852 T437 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368952137924/collection1
[junit4:junit4] 2> 85852 T437 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 85854 T437 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 85854 T437 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 85856 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 85856 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 85857 T437 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368952137924/collection1/'
[junit4:junit4] 2> 85859 T437 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368952137924/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 85859 T437 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368952137924/collection1/lib/README' to classloader
[junit4:junit4] 2> 85868 T416 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 85868 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 85911 T437 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 85974 T437 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 86076 T437 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 86083 T437 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 86675 T437 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 86679 T437 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 86682 T437 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 86688 T437 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 86728 T437 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 86729 T437 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368952137924/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty1/
[junit4:junit4] 2> 86729 T437 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d70b8c3
[junit4:junit4] 2> 86730 T437 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 86731 T437 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty1
[junit4:junit4] 2> 86731 T437 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty1/index/
[junit4:junit4] 2> 86731 T437 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 86732 T437 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty1/index
[junit4:junit4] 2> 86736 T437 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1b9907f7 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1c7d1466),segFN=segments_1,generation=1}
[junit4:junit4] 2> 86736 T437 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 86739 T437 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 86740 T437 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 86741 T437 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 86742 T437 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 86742 T437 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 86743 T437 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 86743 T437 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 86744 T437 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 86745 T437 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 86758 T437 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 86764 T437 oass.SolrIndexSearcher.<init> Opening Searcher@56564a1f main
[junit4:junit4] 2> 86765 T437 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty1/tlog
[junit4:junit4] 2> 86766 T437 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 86766 T437 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 86772 T438 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@56564a1f main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 86775 T437 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 86775 T437 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 87369 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 87370 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43350_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43350/zx/cp"}
[junit4:junit4] 2> 87370 T417 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4:junit4] 2> 87371 T417 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 87385 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 87385 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 87385 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 87779 T437 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 87779 T437 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43350/zx/cp collection:collection1 shard:shard1
[junit4:junit4] 2> 87780 T437 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 87799 T437 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 87819 T437 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 87820 T437 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 87820 T437 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:43350/zx/cp/collection1/
[junit4:junit4] 2> 87821 T437 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 87821 T437 oasc.SyncStrategy.syncToMe http://127.0.0.1:43350/zx/cp/collection1/ has no replicas
[junit4:junit4] 2> 87821 T437 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:43350/zx/cp/collection1/
[junit4:junit4] 2> 87822 T437 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 88894 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 88915 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 88915 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 88915 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 88961 T437 oasc.ZkController.register We are http://127.0.0.1:43350/zx/cp/collection1/ and leader is http://127.0.0.1:43350/zx/cp/collection1/
[junit4:junit4] 2> 88961 T437 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43350/zx/cp
[junit4:junit4] 2> 88961 T437 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 88962 T437 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 88962 T437 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 88965 T437 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 88966 T394 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 88967 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 88968 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 89212 T394 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 89216 T394 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43353
[junit4:junit4] 2> 89217 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 89217 T394 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 89218 T394 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368952142509
[junit4:junit4] 2> 89219 T394 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368952142509/solr.xml
[junit4:junit4] 2> 89219 T394 oasc.CoreContainer.<init> New CoreContainer 1856862618
[junit4:junit4] 2> 89220 T394 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368952142509/'
[junit4:junit4] 2> 89221 T394 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368952142509/'
[junit4:junit4] 2> 89351 T394 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 89352 T394 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 89353 T394 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 89353 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 89354 T394 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 89354 T394 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 89355 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 89355 T394 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 89356 T394 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 89357 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 89375 T394 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 89376 T394 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43343/solr
[junit4:junit4] 2> 89376 T394 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 89377 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 89381 T450 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4542ca0 name:ZooKeeperConnection Watcher:127.0.0.1:43343 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 89382 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 89397 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 89412 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 89415 T452 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e27bb6 name:ZooKeeperConnection Watcher:127.0.0.1:43343/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 89415 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 89429 T394 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 90425 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 90427 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43350_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43350/zx/cp"}
[junit4:junit4] 2> 90450 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 90450 T394 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43353_zx%2Fcp
[junit4:junit4] 2> 90450 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 90450 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 90450 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 90453 T394 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43353_zx%2Fcp
[junit4:junit4] 2> 90464 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 90464 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 90465 T436 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 90466 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 90466 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 90468 T423 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 90468 T452 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 90470 T416 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 90477 T453 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368952142509/collection1
[junit4:junit4] 2> 90477 T453 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 90478 T453 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 90478 T453 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 90480 T453 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368952142509/collection1/'
[junit4:junit4] 2> 90482 T453 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368952142509/collection1/lib/README' to classloader
[junit4:junit4] 2> 90482 T453 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368952142509/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 90533 T453 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 90594 T453 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 90696 T453 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 90703 T453 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 91273 T453 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 91276 T453 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 91278 T453 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 91283 T453 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 91313 T453 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 91313 T453 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368952142509/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty2/
[junit4:junit4] 2> 91313 T453 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d70b8c3
[junit4:junit4] 2> 91314 T453 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 91315 T453 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty2
[junit4:junit4] 2> 91315 T453 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty2/index/
[junit4:junit4] 2> 91315 T453 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 91316 T453 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty2/index
[junit4:junit4] 2> 91318 T453 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@2574efb4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4725931e),segFN=segments_1,generation=1}
[junit4:junit4] 2> 91318 T453 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 91321 T453 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 91321 T453 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 91322 T453 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 91322 T453 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 91323 T453 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 91323 T453 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 91323 T453 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 91324 T453 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 91324 T453 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 91334 T453 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 91339 T453 oass.SolrIndexSearcher.<init> Opening Searcher@3b11ed3a main
[junit4:junit4] 2> 91340 T453 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty2/tlog
[junit4:junit4] 2> 91340 T453 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 91341 T453 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 91345 T454 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3b11ed3a main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 91347 T453 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 91347 T453 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 91960 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 91962 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43353_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43353/zx/cp"}
[junit4:junit4] 2> 91962 T417 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 91963 T417 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 92004 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 92005 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 92005 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 92004 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 92378 T453 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 92378 T453 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43353/zx/cp collection:collection1 shard:shard2
[junit4:junit4] 2> 92380 T453 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 92402 T453 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 92411 T453 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 92411 T453 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 92411 T453 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:43353/zx/cp/collection1/
[junit4:junit4] 2> 92412 T453 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 92412 T453 oasc.SyncStrategy.syncToMe http://127.0.0.1:43353/zx/cp/collection1/ has no replicas
[junit4:junit4] 2> 92412 T453 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:43353/zx/cp/collection1/
[junit4:junit4] 2> 92413 T453 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 93512 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 93536 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 93536 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 93536 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 93536 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 93551 T453 oasc.ZkController.register We are http://127.0.0.1:43353/zx/cp/collection1/ and leader is http://127.0.0.1:43353/zx/cp/collection1/
[junit4:junit4] 2> 93551 T453 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43353/zx/cp
[junit4:junit4] 2> 93551 T453 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 93552 T453 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 93552 T453 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 93555 T453 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 93556 T394 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 93557 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 93558 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 93807 T394 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 93810 T394 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43357
[junit4:junit4] 2> 93811 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 93811 T394 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 93812 T394 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368952147102
[junit4:junit4] 2> 93812 T394 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368952147102/solr.xml
[junit4:junit4] 2> 93813 T394 oasc.CoreContainer.<init> New CoreContainer 1243156721
[junit4:junit4] 2> 93814 T394 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368952147102/'
[junit4:junit4] 2> 93814 T394 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368952147102/'
[junit4:junit4] 2> 93923 T394 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 93924 T394 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 93925 T394 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 93925 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 93926 T394 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 93926 T394 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 93927 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 93927 T394 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 93927 T394 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 93928 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 93944 T394 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 93945 T394 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43343/solr
[junit4:junit4] 2> 93945 T394 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 93946 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 93950 T466 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7139b2a2 name:ZooKeeperConnection Watcher:127.0.0.1:43343 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 93951 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 93968 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 93981 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 93984 T468 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3aa5378 name:ZooKeeperConnection Watcher:127.0.0.1:43343/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 93984 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 93991 T394 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 94998 T394 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43357_zx%2Fcp
[junit4:junit4] 2> 95028 T394 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43357_zx%2Fcp
[junit4:junit4] 2> 95032 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 95032 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 95032 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 95033 T468 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 95033 T436 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 95034 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 95036 T452 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 95036 T416 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 95077 T423 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 95080 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 95081 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43353_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43353/zx/cp"}
[junit4:junit4] 2> 95083 T469 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368952147102/collection1
[junit4:junit4] 2> 95084 T469 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 95085 T469 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 95085 T469 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 95087 T469 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368952147102/collection1/'
[junit4:junit4] 2> 95090 T469 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368952147102/collection1/lib/README' to classloader
[junit4:junit4] 2> 95090 T469 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368952147102/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 95127 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 95127 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 95127 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 95127 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 95127 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 95167 T469 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 95282 T469 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 95384 T469 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 95396 T469 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 96108 T469 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 96112 T469 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 96115 T469 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 96123 T469 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 96164 T469 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 96164 T469 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368952147102/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/
[junit4:junit4] 2> 96165 T469 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d70b8c3
[junit4:junit4] 2> 96166 T469 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 96166 T469 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3
[junit4:junit4] 2> 96167 T469 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/index/
[junit4:junit4] 2> 96167 T469 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 96168 T469 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/index
[junit4:junit4] 2> 96171 T469 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@25cb7796 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1699fbc5),segFN=segments_1,generation=1}
[junit4:junit4] 2> 96172 T469 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 96175 T469 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 96175 T469 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 96176 T469 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 96177 T469 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 96178 T469 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 96178 T469 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 96179 T469 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 96180 T469 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 96180 T469 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 96200 T469 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 96209 T469 oass.SolrIndexSearcher.<init> Opening Searcher@7397be10 main
[junit4:junit4] 2> 96209 T469 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/tlog
[junit4:junit4] 2> 96210 T469 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 96211 T469 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 96217 T470 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7397be10 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 96219 T469 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 96220 T469 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 96637 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 96639 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43357_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43357/zx/cp"}
[junit4:junit4] 2> 96639 T417 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 96639 T417 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 96654 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 96655 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 96655 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 96655 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 96655 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 97223 T469 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 97223 T469 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43357/zx/cp collection:collection1 shard:shard1
[junit4:junit4] 2> 97231 T469 oasc.ZkController.register We are http://127.0.0.1:43357/zx/cp/collection1/ and leader is http://127.0.0.1:43350/zx/cp/collection1/
[junit4:junit4] 2> 97232 T469 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43357/zx/cp
[junit4:junit4] 2> 97232 T469 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 97232 T469 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C32 name=collection1 org.apache.solr.core.SolrCore@48a3743 url=http://127.0.0.1:43357/zx/cp/collection1 node=127.0.0.1:43357_zx%2Fcp C32_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:43357_zx%2Fcp, base_url=http://127.0.0.1:43357/zx/cp}
[junit4:junit4] 2> 97240 T471 C32 P43357 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 97241 T469 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 97241 T471 C32 P43357 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 97242 T471 C32 P43357 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 97243 T471 C32 P43357 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 97243 T394 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 97244 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 97245 T471 C32 P43357 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 97245 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 97289 T426 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 97524 T394 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 97528 T394 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:43361
[junit4:junit4] 2> 97529 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 97529 T394 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 97530 T394 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368952150799
[junit4:junit4] 2> 97531 T394 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368952150799/solr.xml
[junit4:junit4] 2> 97531 T394 oasc.CoreContainer.<init> New CoreContainer 232166480
[junit4:junit4] 2> 97532 T394 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368952150799/'
[junit4:junit4] 2> 97533 T394 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368952150799/'
[junit4:junit4] 2> 97662 T394 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 97663 T394 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 97664 T394 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 97665 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 97666 T394 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 97666 T394 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 97667 T394 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 97668 T394 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 97668 T394 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 97669 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 97688 T394 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 97689 T394 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:43343/solr
[junit4:junit4] 2> 97690 T394 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 97691 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 97694 T483 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@405e806f name:ZooKeeperConnection Watcher:127.0.0.1:43343 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 97695 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 97708 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 97723 T394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 97726 T485 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3597b40d name:ZooKeeperConnection Watcher:127.0.0.1:43343/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 97726 T394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 97739 T394 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 98163 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 98164 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43357_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43357/zx/cp"}
[junit4:junit4] 2> 98179 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98179 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98179 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98179 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98179 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98179 T485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98290 T426 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 98290 T426 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=4&state=recovering&nodeName=127.0.0.1:43357_zx%252Fcp&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1002
[junit4:junit4] 2> 98746 T394 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43361_zx%2Fcp
[junit4:junit4] 2> 98750 T394 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43361_zx%2Fcp
[junit4:junit4] 2> 98753 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98753 T485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98753 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98753 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 98754 T436 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 98754 T468 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 98755 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 98756 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 98756 T416 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 98763 T485 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 98763 T452 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 98763 T423 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 98770 T486 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368952150799/collection1
[junit4:junit4] 2> 98770 T486 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 98776 T486 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 98776 T486 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 98779 T486 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368952150799/collection1/'
[junit4:junit4] 2> 98780 T486 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368952150799/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 98781 T486 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368952150799/collection1/lib/README' to classloader
[junit4:junit4] 2> 98842 T486 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 98907 T486 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 99009 T486 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 99016 T486 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 99675 T486 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 99679 T486 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 99682 T486 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 99688 T486 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 99729 T486 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 99729 T486 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368952150799/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/
[junit4:junit4] 2> 99730 T486 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1d70b8c3
[junit4:junit4] 2> 99731 T486 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 99731 T486 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4
[junit4:junit4] 2> 99732 T486 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index/
[junit4:junit4] 2> 99732 T486 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 99733 T486 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index
[junit4:junit4] 2> 99736 T486 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7244499b lockFactory=org.apache.lucene.store.NativeFSLockFactory@517ab786),segFN=segments_1,generation=1}
[junit4:junit4] 2> 99736 T486 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 99739 T486 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 99740 T486 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 99740 T486 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 99741 T486 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 99742 T486 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 99742 T486 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 99743 T486 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 99744 T486 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 99744 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 99762 T486 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 99768 T486 oass.SolrIndexSearcher.<init> Opening Searcher@2b02a3ad main
[junit4:junit4] 2> 99768 T486 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/tlog
[junit4:junit4] 2> 99769 T486 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 99770 T486 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 99774 T487 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2b02a3ad main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 99777 T486 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 99777 T486 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> C32_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:43357_zx%2Fcp, base_url=http://127.0.0.1:43357/zx/cp}
[junit4:junit4] 2> 100293 T471 C32 P43357 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:43350/zx/cp/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 100302 T471 C32 P43357 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&connTimeout=30000&socketTimeout=30000&retry=false
[junit4:junit4] 2> 100325 T471 C32 P43357 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:43357/zx/cp START replicas=[http://127.0.0.1:43350/zx/cp/collection1/] nUpdates=100
[junit4:junit4] 2> 100335 T471 C32 P43357 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 100336 T471 C32 P43357 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 100336 T471 C32 P43357 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 100336 T471 C32 P43357 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 100336 T471 C32 P43357 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 100337 T471 C32 P43357 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:43350/zx/cp/collection1/. core=collection1
[junit4:junit4] 2> 100337 T471 C32 P43357 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C33 name=collection1 org.apache.solr.core.SolrCore@7fc8a3e1 url=http://127.0.0.1:43350/zx/cp/collection1 node=127.0.0.1:43350_zx%2Fcp C33_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:43350_zx%2Fcp, base_url=http://127.0.0.1:43350/zx/cp, leader=true}
[junit4:junit4] 2> 100353 T427 C33 P43350 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 100372 T428 C33 P43350 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 100375 T428 C33 P43350 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1b9907f7 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1c7d1466),segFN=segments_1,generation=1}
[junit4:junit4] 2> 100376 T428 C33 P43350 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 100377 T428 C33 P43350 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1b9907f7 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1c7d1466),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1b9907f7 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1c7d1466),segFN=segments_2,generation=2}
[junit4:junit4] 2> 100378 T428 C33 P43350 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 100378 T428 C33 P43350 oass.SolrIndexSearcher.<init> Opening Searcher@6d52ca09 realtime
[junit4:junit4] 2> 100379 T428 C33 P43350 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 100380 T428 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 27
[junit4:junit4] 2> 100381 T471 C32 P43357 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 100382 T471 C32 P43357 oascsi.HttpClientUtil.createClient Creating new http client, config:connTimeout=5000&socketTimeout=20000&allowCompression=false&maxConnections=10000&maxConnectionsPerHost=10000
[junit4:junit4] 2> 100403 T471 C32 P43357 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 100406 T429 C33 P43350 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 100407 T429 C33 P43350 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2
[junit4:junit4] 2> 100409 T471 C32 P43357 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 100409 T471 C32 P43357 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 100410 T471 C32 P43357 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 100414 T429 C33 P43350 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 100416 T471 C32 P43357 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 100417 T471 C32 P43357 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/index.20130519052913942
[junit4:junit4] 2> 100418 T471 C32 P43357 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@6114cd31 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1be0c344) fullCopy=false
[junit4:junit4] 2> 100431 T429 C33 P43350 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=4
[junit4:junit4] 2> 100444 T471 C32 P43357 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 100447 T471 C32 P43357 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 100447 T471 C32 P43357 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 100454 T471 C32 P43357 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@25cb7796 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1699fbc5),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@25cb7796 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1699fbc5),segFN=segments_2,generation=2}
[junit4:junit4] 2> 100455 T471 C32 P43357 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 100455 T471 C32 P43357 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 100456 T471 C32 P43357 oass.SolrIndexSearcher.<init> Opening Searcher@7c698b24 main
[junit4:junit4] 2> 100457 T470 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7c698b24 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 100458 T471 C32 P43357 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/index.20130519052913942 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/index.20130519052913942;done=true>>]
[junit4:junit4] 2> 100458 T471 C32 P43357 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/index.20130519052913942
[junit4:junit4] 2> 100459 T471 C32 P43357 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty3/index.20130519052913942
[junit4:junit4] 2> 100459 T471 C32 P43357 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 100459 T471 C32 P43357 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 100460 T471 C32 P43357 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 100460 T471 C32 P43357 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 100466 T471 C32 P43357 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 101189 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 101191 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43361_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43361/zx/cp"}
[junit4:junit4] 2> 101191 T417 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 101191 T417 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 101223 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43357_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43357/zx/cp"}
[junit4:junit4] 2> 101236 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 101236 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 101236 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 101236 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 101236 T485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 101236 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 101782 T486 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 101782 T486 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:43361/zx/cp collection:collection1 shard:shard2
[junit4:junit4] 2> 101812 T486 oasc.ZkController.register We are http://127.0.0.1:43361/zx/cp/collection1/ and leader is http://127.0.0.1:43353/zx/cp/collection1/
[junit4:junit4] 2> 101812 T486 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:43361/zx/cp
[junit4:junit4] 2> 101813 T486 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 101813 T486 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C34 name=collection1 org.apache.solr.core.SolrCore@2c46465f url=http://127.0.0.1:43361/zx/cp/collection1 node=127.0.0.1:43361_zx%2Fcp C34_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:43361_zx%2Fcp, base_url=http://127.0.0.1:43361/zx/cp}
[junit4:junit4] 2> 101813 T490 C34 P43361 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 101815 T490 C34 P43361 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 101815 T486 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 101815 T490 C34 P43361 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 101816 T490 C34 P43361 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 101817 T394 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 101817 T394 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 101817 T490 C34 P43361 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 101818 T394 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 101832 T394 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 101833 T442 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 101834 T394 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 15 for each attempt
[junit4:junit4] 2> 101835 T394 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4:junit4] 2> 101836 T394 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 102750 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 102752 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43361_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43361/zx/cp"}
[junit4:junit4] 2> 102769 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 102769 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 102769 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 102769 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 102769 T485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 102769 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 102834 T442 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4:junit4] 2> 102834 T442 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:43361_zx%252Fcp&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1001
[junit4:junit4] 2> 102838 T394 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 103842 T394 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> C34_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:43361_zx%2Fcp, base_url=http://127.0.0.1:43361/zx/cp}
[junit4:junit4] 2> 104836 T490 C34 P43361 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:43353/zx/cp/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 104836 T490 C34 P43361 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:43361/zx/cp START replicas=[http://127.0.0.1:43353/zx/cp/collection1/] nUpdates=100
[junit4:junit4] 2> 104838 T490 C34 P43361 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 104838 T490 C34 P43361 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 104839 T490 C34 P43361 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 104839 T490 C34 P43361 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 104839 T490 C34 P43361 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 104840 T490 C34 P43361 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:43353/zx/cp/collection1/. core=collection1
[junit4:junit4] 2> 104840 T490 C34 P43361 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 104845 T394 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> ASYNC NEW_CORE C35 name=collection1 org.apache.solr.core.SolrCore@5ae06589 url=http://127.0.0.1:43353/zx/cp/collection1 node=127.0.0.1:43353_zx%2Fcp C35_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:43353_zx%2Fcp, base_url=http://127.0.0.1:43353/zx/cp, leader=true}
[junit4:junit4] 2> 104847 T443 C35 P43353 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 104853 T444 C35 P43353 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 104856 T444 C35 P43353 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@2574efb4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4725931e),segFN=segments_1,generation=1}
[junit4:junit4] 2> 104857 T444 C35 P43353 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 104858 T444 C35 P43353 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@2574efb4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4725931e),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@2574efb4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4725931e),segFN=segments_2,generation=2}
[junit4:junit4] 2> 104858 T444 C35 P43353 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 104859 T444 C35 P43353 oass.SolrIndexSearcher.<init> Opening Searcher@1d15c906 realtime
[junit4:junit4] 2> 104860 T444 C35 P43353 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 104860 T444 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 7
[junit4:junit4] 2> 104861 T490 C34 P43361 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 104862 T490 C34 P43361 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 104864 T445 C35 P43353 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 104865 T445 C35 P43353 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=1
[junit4:junit4] 2> 104866 T490 C34 P43361 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 104866 T490 C34 P43361 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 104866 T490 C34 P43361 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 104870 T445 C35 P43353 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 104871 T490 C34 P43361 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 104872 T490 C34 P43361 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index.20130519052918397
[junit4:junit4] 2> 104873 T490 C34 P43361 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@589e3112 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7de37ee4) fullCopy=false
[junit4:junit4] 2> 104876 T445 C35 P43353 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 104877 T490 C34 P43361 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 104878 T490 C34 P43361 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 104878 T490 C34 P43361 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 104880 T490 C34 P43361 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7244499b lockFactory=org.apache.lucene.store.NativeFSLockFactory@517ab786),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7244499b lockFactory=org.apache.lucene.store.NativeFSLockFactory@517ab786),segFN=segments_2,generation=2}
[junit4:junit4] 2> 104881 T490 C34 P43361 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 104881 T490 C34 P43361 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 104882 T490 C34 P43361 oass.SolrIndexSearcher.<init> Opening Searcher@7dcea13b main
[junit4:junit4] 2> 104883 T487 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7dcea13b main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 104884 T490 C34 P43361 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index.20130519052918397 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index.20130519052918397;done=true>>]
[junit4:junit4] 2> 104884 T490 C34 P43361 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index.20130519052918397
[junit4:junit4] 2> 104884 T490 C34 P43361 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index.20130519052918397
[junit4:junit4] 2> 104885 T490 C34 P43361 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 104885 T490 C34 P43361 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 104885 T490 C34 P43361 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 104885 T490 C34 P43361 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 104888 T490 C34 P43361 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 105779 T417 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 105781 T417 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:43361_zx%2Fcp",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:43361/zx/cp"}
[junit4:junit4] 2> 105790 T416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 105791 T452 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 105791 T423 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 105791 T436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 105791 T485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 105791 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 105847 T394 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 105849 T394 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
[junit4:junit4] 2> ASYNC NEW_CORE C36 name=collection1 org.apache.solr.core.SolrCore@7db45d04 url=http://127.0.0.1:43346/zx/cp/collection1 node=127.0.0.1:43346_zx%2Fcp C36_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:43346_zx%2Fcp, base_url=http://127.0.0.1:43346/zx/cp, leader=true}
[junit4:junit4] 2> 105859 T406 C36 P43346 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 105863 T406 C36 P43346 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@ac49d10 lockFactory=org.apache.lucene.store.NativeFSLockFactory@8c2cbff),segFN=segments_1,generation=1}
[junit4:junit4] 2> 105863 T406 C36 P43346 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 105864 T406 C36 P43346 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@ac49d10 lockFactory=org.apache.lucene.store.NativeFSLockFactory@8c2cbff),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@ac49d10 lockFactory=org.apache.lucene.store.NativeFSLockFactory@8c2cbff),segFN=segments_2,generation=2}
[junit4:junit4] 2> 105865 T406 C36 P43346 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 105866 T406 C36 P43346 oass.SolrIndexSearcher.<init> Opening Searcher@8b49eac main
[junit4:junit4] 2> 105866 T406 C36 P43346 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 105867 T420 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@8b49eac main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 105868 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 9
[junit4:junit4] 2> 105895 T430 C33 P43350 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 105896 T430 C33 P43350 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1b9907f7 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1c7d1466),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1b9907f7 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1c7d1466),segFN=segments_3,generation=3}
[junit4:junit4] 2> 105897 T430 C33 P43350 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 105898 T430 C33 P43350 oass.SolrIndexSearcher.<init> Opening Searcher@540e4427 main
[junit4:junit4] 2> 105898 T430 C33 P43350 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 105899 T438 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@540e4427 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 105914 T430 C33 P43350 oasu.SolrCmdDistributor.distribCommit Distrib commit to:[StdNode: http://127.0.0.1:43357/zx/cp/collection1/, StdNode: http://127.0.0.1:43353/zx/cp/collection1/, StdNode: http://127.0.0.1:43361/zx/cp/collection1/] params:commit_end_point=true&commit=true&softCommit=false&waitSearcher=true&expungeDeletes=false
[junit4:junit4] 2> 105924 T446 C35 P43353 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 105925 T446 C35 P43353 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@2574efb4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4725931e),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@2574efb4 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4725931e),segFN=segments_3,generation=3}
[junit4:junit4] 2> 105925 T446 C35 P43353 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 105926 T446 C35 P43353 oass.SolrIndexSearcher.<init> Opening Searcher@5f317c78 main
[junit4:junit4] 2> 105927 T446 C35 P43353 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 105928 T454 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5f317c78 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 105928 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 5
[junit4:junit4] 2> C32_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:43357_zx%2Fcp, base_url=http://127.0.0.1:43357/zx/cp}
[junit4:junit4] 2> 105932 T458 C32 P43357 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> C34_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:43361_zx%2Fcp, base_url=http://127.0.0.1:43361/zx/cp}
[junit4:junit4] 2> 105932 T475 C34 P43361 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 105933 T458 C32 P43357 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@25cb7796 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1699fbc5),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@25cb7796 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1699fbc5),segFN=segments_3,generation=3}
[junit4:junit4] 2> 105934 T475 C34 P43361 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7244499b lockFactory=org.apache.lucene.store.NativeFSLockFactory@517ab786),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7244499b lockFactory=org.apache.lucene.store.NativeFSLockFactory@517ab786),segFN=segments_3,generation=3}
[junit4:junit4] 2> 105934 T458 C32 P43357 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 105934 T475 C34 P43361 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 105935 T458 C32 P43357 oass.SolrIndexSearcher.<init> Opening Searcher@4efbd2cc main
[junit4:junit4] 2> 105936 T475 C34 P43361 oass.SolrIndexSearcher.<init> Opening Searcher@52b15537 main
[junit4:junit4] 2> 105936 T458 C32 P43357 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 105937 T475 C34 P43361 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 105937 T470 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4efbd2cc main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 105938 T487 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@52b15537 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 105938 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 7
[junit4:junit4] 2> 105938 T475 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 7
[junit4:junit4] 2> 105940 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 45
[junit4:junit4] 2> 105941 T394 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 105946 T431 C33 P43350 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 105950 T459 C32 P43357 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1
[junit4:junit4] 2> 105954 T447 C35 P43353 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1
[junit4:junit4] 2> 105957 T476 C34 P43361 oasc.SolrCore.execute [collection1] webapp=/zx/cp path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 107968 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1435450381688635392)} 0 6
[junit4:junit4] 2> 107988 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={update.distrib=FROMLEADER&_version_=-1435450381694926848&update.from=http://127.0.0.1:43353/zx/cp/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1435450381694926848)} 0 2
[junit4:junit4] 2> 107993 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={update.distrib=FROMLEADER&_version_=-1435450381708558336&update.from=http://127.0.0.1:43350/zx/cp/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1435450381708558336)} 0 2
[junit4:junit4] 2> 107994 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={update.distrib=TOLEADER&wt=javabin&version=2} {deleteByQuery=*:* (-1435450381708558336)} 0 8
[junit4:junit4] 2> 107995 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1435450381694926848)} 0 23
[junit4:junit4] 2> 108000 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[0 (1435450381722189824)]} 0 1
[junit4:junit4] 2> 108022 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[0 (1435450381727432704)]} 0 1
[junit4:junit4] 2> 108023 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[0 (1435450381727432704)]} 0 19
[junit4:junit4] 2> 108027 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[1 (1435450381751549952)]} 0 0
[junit4:junit4] 2> 108037 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[1 (1435450381756792832)]} 0 1
[junit4:junit4] 2> 108038 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[1 (1435450381756792832)]} 0 6
[junit4:junit4] 2> 108043 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[2 (1435450381767278592)]} 0 1
[junit4:junit4] 2> 108052 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[2 (1435450381772521472)]} 0 1
[junit4:junit4] 2> 108053 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[2 (1435450381772521472)]} 0 6
[junit4:junit4] 2> 108057 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[3 (1435450381783007232)]} 0 0
[junit4:junit4] 2> 108067 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[3 (1435450381788250112)]} 0 1
[junit4:junit4] 2> 108068 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[3 (1435450381788250112)]} 0 6
[junit4:junit4] 2> 108072 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[4 (1435450381798735872)]} 0 0
[junit4:junit4] 2> 108093 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[4 (1435450381814464512)]} 0 1
[junit4:junit4] 2> 108094 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[4 (1435450381814464512)]} 0 7
[junit4:junit4] 2> 108095 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[4]} 0 18
[junit4:junit4] 2> 108100 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[5 (1435450381827047424)]} 0 1
[junit4:junit4] 2> 108109 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[5 (1435450381832290304)]} 0 1
[junit4:junit4] 2> 108110 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[5 (1435450381832290304)]} 0 6
[junit4:junit4] 2> 108114 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[6 (1435450381842776064)]} 0 0
[junit4:junit4] 2> 108123 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[6 (1435450381846970368)]} 0 1
[junit4:junit4] 2> 108124 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[6 (1435450381846970368)]} 0 6
[junit4:junit4] 2> 108129 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[7 (1435450381857456128)]} 0 1
[junit4:junit4] 2> 108143 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[7 (1435450381866893312)]} 0 1
[junit4:junit4] 2> 108144 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[7 (1435450381866893312)]} 0 7
[junit4:junit4] 2> 108145 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[7]} 0 12
[junit4:junit4] 2> 108150 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[8 (1435450381879476224)]} 0 1
[junit4:junit4] 2> 108162 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[8 (1435450381887864832)]} 0 1
[junit4:junit4] 2> 108163 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[8 (1435450381887864832)]} 0 6
[junit4:junit4] 2> 108164 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[8]} 0 10
[junit4:junit4] 2> 108170 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[9 (1435450381900447744)]} 0 1
[junit4:junit4] 2> 108179 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[9 (1435450381905690624)]} 0 1
[junit4:junit4] 2> 108180 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[9 (1435450381905690624)]} 0 6
[junit4:junit4] 2> 108185 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[10 (1435450381916176384)]} 0 1
[junit4:junit4] 2> 108197 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[10 (1435450381924564992)]} 0 1
[junit4:junit4] 2> 108198 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[10 (1435450381924564992)]} 0 6
[junit4:junit4] 2> 108199 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[10]} 0 10
[junit4:junit4] 2> 108203 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[11 (1435450381936099328)]} 0 0
[junit4:junit4] 2> 108213 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[11 (1435450381941342208)]} 0 1
[junit4:junit4] 2> 108214 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[11 (1435450381941342208)]} 0 6
[junit4:junit4] 2> 108218 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[12 (1435450381951827968)]} 0 0
[junit4:junit4] 2> 108227 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[12 (1435450381956022272)]} 0 1
[junit4:junit4] 2> 108228 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[12 (1435450381956022272)]} 0 6
[junit4:junit4] 2> 108232 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[13 (1435450381966508032)]} 0 0
[junit4:junit4] 2> 108245 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[13 (1435450381974896640)]} 0 1
[junit4:junit4] 2> 108246 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[13 (1435450381974896640)]} 0 6
[junit4:junit4] 2> 108247 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[13]} 0 10
[junit4:junit4] 2> 108252 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[14 (1435450381986430976)]} 0 1
[junit4:junit4] 2> 108264 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[14 (1435450381994819584)]} 0 1
[junit4:junit4] 2> 108265 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[14 (1435450381994819584)]} 0 6
[junit4:junit4] 2> 108266 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[14]} 0 10
[junit4:junit4] 2> 108270 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[15 (1435450382006353920)]} 0 1
[junit4:junit4] 2> 108279 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[15 (1435450382010548224)]} 0 1
[junit4:junit4] 2> 108279 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[15 (1435450382010548224)]} 0 5
[junit4:junit4] 2> 108284 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[16 (1435450382021033984)]} 0 1
[junit4:junit4] 2> 108293 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[16 (1435450382025228288)]} 0 1
[junit4:junit4] 2> 108294 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[16 (1435450382025228288)]} 0 6
[junit4:junit4] 2> 108298 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[17 (1435450382035714048)]} 0 0
[junit4:junit4] 2> 108310 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[17 (1435450382043054080)]} 0 1
[junit4:junit4] 2> 108311 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[17 (1435450382043054080)]} 0 6
[junit4:junit4] 2> 108312 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[17]} 0 10
[junit4:junit4] 2> 108317 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[18 (1435450382054588416)]} 0 1
[junit4:junit4] 2> 108329 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[18 (1435450382062977024)]} 0 1
[junit4:junit4] 2> 108330 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[18 (1435450382062977024)]} 0 6
[junit4:junit4] 2> 108331 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[18]} 0 10
[junit4:junit4] 2> 108335 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[19 (1435450382074511360)]} 0 1
[junit4:junit4] 2> 108349 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[19 (1435450382083948544)]} 0 1
[junit4:junit4] 2> 108350 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[19 (1435450382083948544)]} 0 6
[junit4:junit4] 2> 108351 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[19]} 0 11
[junit4:junit4] 2> 108355 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[20 (1435450382095482880)]} 0 0
[junit4:junit4] 2> 108365 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[20 (1435450382100725760)]} 0 1
[junit4:junit4] 2> 108366 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[20 (1435450382100725760)]} 0 6
[junit4:junit4] 2> 108370 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[21 (1435450382111211520)]} 0 1
[junit4:junit4] 2> 108379 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[21 (1435450382115405824)]} 0 1
[junit4:junit4] 2> 108380 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[21 (1435450382115405824)]} 0 7
[junit4:junit4] 2> 108384 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[22 (1435450382125891584)]} 0 1
[junit4:junit4] 2> 108397 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[22 (1435450382134280192)]} 0 1
[junit4:junit4] 2> 108398 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[22 (1435450382134280192)]} 0 6
[junit4:junit4] 2> 108399 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[22]} 0 10
[junit4:junit4] 2> 108404 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[23 (1435450382145814528)]} 0 1
[junit4:junit4] 2> 108417 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[23 (1435450382155251712)]} 0 1
[junit4:junit4] 2> 108418 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[23 (1435450382155251712)]} 0 6
[junit4:junit4] 2> 108419 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[23]} 0 11
[junit4:junit4] 2> 108424 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[24 (1435450382166786048)]} 0 1
[junit4:junit4] 2> 108437 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[24 (1435450382176223232)]} 0 1
[junit4:junit4] 2> 108438 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[24 (1435450382176223232)]} 0 6
[junit4:junit4] 2> 108439 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[24]} 0 11
[junit4:junit4] 2> 108444 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[25 (1435450382187757568)]} 0 1
[junit4:junit4] 2> 108453 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[25 (1435450382193000448)]} 0 1
[junit4:junit4] 2> 108454 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[25 (1435450382193000448)]} 0 6
[junit4:junit4] 2> 108458 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[26 (1435450382203486208)]} 0 0
[junit4:junit4] 2> 108471 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[26 (1435450382211874816)]} 0 1
[junit4:junit4] 2> 108472 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[26 (1435450382211874816)]} 0 6
[junit4:junit4] 2> 108473 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[26]} 0 11
[junit4:junit4] 2> 108477 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[27 (1435450382223409152)]} 0 1
[junit4:junit4] 2> 108489 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[27 (1435450382230749184)]} 0 1
[junit4:junit4] 2> 108490 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[27 (1435450382230749184)]} 0 6
[junit4:junit4] 2> 108491 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[27]} 0 10
[junit4:junit4] 2> 108495 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[28 (1435450382242283520)]} 0 0
[junit4:junit4] 2> 108508 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[28 (1435450382250672128)]} 0 1
[junit4:junit4] 2> 108509 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[28 (1435450382250672128)]} 0 6
[junit4:junit4] 2> 108510 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[28]} 0 10
[junit4:junit4] 2> 108514 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[29 (1435450382262206464)]} 0 0
[junit4:junit4] 2> 108528 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[29 (1435450382271643648)]} 0 1
[junit4:junit4] 2> 108529 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[29 (1435450382271643648)]} 0 6
[junit4:junit4] 2> 108530 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[29]} 0 11
[junit4:junit4] 2> 108535 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[30 (1435450382283177984)]} 0 1
[junit4:junit4] 2> 108547 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[30 (1435450382291566592)]} 0 1
[junit4:junit4] 2> 108548 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[30 (1435450382291566592)]} 0 6
[junit4:junit4] 2> 108549 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[30]} 0 10
[junit4:junit4] 2> 108553 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[31 (1435450382303100928)]} 0 1
[junit4:junit4] 2> 108566 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[31 (1435450382311489536)]} 0 1
[junit4:junit4] 2> 108567 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[31 (1435450382311489536)]} 0 6
[junit4:junit4] 2> 108568 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[31]} 0 11
[junit4:junit4] 2> 108573 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[32 (1435450382323023872)]} 0 1
[junit4:junit4] 2> 108582 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[32 (1435450382328266752)]} 0 1
[junit4:junit4] 2> 108583 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[32 (1435450382328266752)]} 0 6
[junit4:junit4] 2> 108587 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[33 (1435450382338752512)]} 0 1
[junit4:junit4] 2> 108595 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[33 (1435450382342946816)]} 0 0
[junit4:junit4] 2> 108597 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[33 (1435450382342946816)]} 0 6
[junit4:junit4] 2> 108601 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[34 (1435450382353432576)]} 0 1
[junit4:junit4] 2> 108613 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[34 (1435450382360772608)]} 0 1
[junit4:junit4] 2> 108614 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[34 (1435450382360772608)]} 0 6
[junit4:junit4] 2> 108615 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[34]} 0 11
[junit4:junit4] 2> 108619 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[35 (1435450382372306944)]} 0 1
[junit4:junit4] 2> 108627 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[35 (1435450382376501248)]} 0 0
[junit4:junit4] 2> 108628 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[35 (1435450382376501248)]} 0 6
[junit4:junit4] 2> 108634 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[36 (1435450382386987008)]} 0 1
[junit4:junit4] 2> 108646 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[36 (1435450382395375616)]} 0 1
[junit4:junit4] 2> 108647 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[36 (1435450382395375616)]} 0 6
[junit4:junit4] 2> 108648 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[36]} 0 11
[junit4:junit4] 2> 108652 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[37 (1435450382406909952)]} 0 1
[junit4:junit4] 2> 108662 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[37 (1435450382412152832)]} 0 1
[junit4:junit4] 2> 108663 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[37 (1435450382412152832)]} 0 6
[junit4:junit4] 2> 108667 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[38 (1435450382422638592)]} 0 1
[junit4:junit4] 2> 108680 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[38 (1435450382431027200)]} 0 1
[junit4:junit4] 2> 108681 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[38 (1435450382431027200)]} 0 6
[junit4:junit4] 2> 108682 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[38]} 0 10
[junit4:junit4] 2> 108687 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[39 (1435450382442561536)]} 0 1
[junit4:junit4] 2> 108699 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[39 (1435450382450950144)]} 0 1
[junit4:junit4] 2> 108700 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[39 (1435450382450950144)]} 0 6
[junit4:junit4] 2> 108701 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[39]} 0 10
[junit4:junit4] 2> 108705 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[40 (1435450382462484480)]} 0 0
[junit4:junit4] 2> 108718 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[40 (1435450382470873088)]} 0 1
[junit4:junit4] 2> 108719 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[40 (1435450382470873088)]} 0 6
[junit4:junit4] 2> 108720 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[40]} 0 11
[junit4:junit4] 2> 108725 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[41 (1435450382482407424)]} 0 1
[junit4:junit4] 2> 108736 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[41 (1435450382490796032)]} 0 0
[junit4:junit4] 2> 108738 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[41 (1435450382490796032)]} 0 6
[junit4:junit4] 2> 108739 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[41]} 0 11
[junit4:junit4] 2> 108743 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[42 (1435450382502330368)]} 0 1
[junit4:junit4] 2> 108752 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[42 (1435450382506524672)]} 0 1
[junit4:junit4] 2> 108753 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[42 (1435450382506524672)]} 0 7
[junit4:junit4] 2> 108757 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[43 (1435450382517010432)]} 0 0
[junit4:junit4] 2> 108770 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[43 (1435450382525399040)]} 0 1
[junit4:junit4] 2> 108771 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[43 (1435450382525399040)]} 0 6
[junit4:junit4] 2> 108772 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[43]} 0 11
[junit4:junit4] 2> 108776 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[44 (1435450382536933376)]} 0 0
[junit4:junit4] 2> 108788 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[44 (1435450382545321984)]} 0 1
[junit4:junit4] 2> 108789 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[44 (1435450382545321984)]} 0 5
[junit4:junit4] 2> 108790 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[44]} 0 10
[junit4:junit4] 2> 108794 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[45 (1435450382555807744)]} 0 1
[junit4:junit4] 2> 108801 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[45 (1435450382560002048)]} 0 1
[junit4:junit4] 2> 108802 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[45 (1435450382560002048)]} 0 5
[junit4:junit4] 2> 108806 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[46 (1435450382568390656)]} 0 1
[junit4:junit4] 2> 108816 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[46 (1435450382573633536)]} 0 1
[junit4:junit4] 2> 108817 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[46 (1435450382573633536)]} 0 6
[junit4:junit4] 2> 108821 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[47 (1435450382584119296)]} 0 1
[junit4:junit4] 2> 108829 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[47 (1435450382588313600)]} 0 0
[junit4:junit4] 2> 108830 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[47 (1435450382588313600)]} 0 6
[junit4:junit4] 2> 108835 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[48 (1435450382598799360)]} 0 1
[junit4:junit4] 2> 108846 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[48 (1435450382606139392)]} 0 0
[junit4:junit4] 2> 108848 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[48 (1435450382606139392)]} 0 6
[junit4:junit4] 2> 108849 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[48]} 0 11
[junit4:junit4] 2> 108853 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[49 (1435450382617673728)]} 0 1
[junit4:junit4] 2> 108866 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[49 (1435450382627110912)]} 0 0
[junit4:junit4] 2> 108867 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[49 (1435450382627110912)]} 0 6
[junit4:junit4] 2> 108868 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[49]} 0 11
[junit4:junit4] 2> 108873 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[50 (1435450382637596672)]} 0 1
[junit4:junit4] 2> 108884 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[50 (1435450382645985280)]} 0 0
[junit4:junit4] 2> 108886 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[50 (1435450382645985280)]} 0 7
[junit4:junit4] 2> 108887 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[50]} 0 11
[junit4:junit4] 2> 108892 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[51 (1435450382657519616)]} 0 1
[junit4:junit4] 2> 108903 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[51 (1435450382665908224)]} 0 1
[junit4:junit4] 2> 108904 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[51 (1435450382665908224)]} 0 6
[junit4:junit4] 2> 108905 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[51]} 0 10
[junit4:junit4] 2> 108909 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[52 (1435450382676393984)]} 0 1
[junit4:junit4] 2> 108920 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[52 (1435450382683734016)]} 0 0
[junit4:junit4] 2> 108922 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[52 (1435450382683734016)]} 0 6
[junit4:junit4] 2> 108922 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[52]} 0 9
[junit4:junit4] 2> 108927 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[53 (1435450382695268352)]} 0 1
[junit4:junit4] 2> 108939 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[53 (1435450382702608384)]} 0 1
[junit4:junit4] 2> 108940 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[53 (1435450382702608384)]} 0 7
[junit4:junit4] 2> 108941 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[53]} 0 11
[junit4:junit4] 2> 108945 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[54 (1435450382714142720)]} 0 0
[junit4:junit4] 2> 108958 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[54 (1435450382722531328)]} 0 1
[junit4:junit4] 2> 108959 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[54 (1435450382722531328)]} 0 6
[junit4:junit4] 2> 108960 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[54]} 0 11
[junit4:junit4] 2> 108964 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[55 (1435450382734065664)]} 0 1
[junit4:junit4] 2> 108975 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[55 (1435450382741405696)]} 0 1
[junit4:junit4] 2> 108976 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[55 (1435450382741405696)]} 0 6
[junit4:junit4] 2> 108977 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[55]} 0 9
[junit4:junit4] 2> 108981 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[56 (1435450382751891456)]} 0 1
[junit4:junit4] 2> 108992 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[56 (1435450382759231488)]} 0 0
[junit4:junit4] 2> 108993 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[56 (1435450382759231488)]} 0 6
[junit4:junit4] 2> 108994 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[56]} 0 10
[junit4:junit4] 2> 108998 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[57 (1435450382769717248)]} 0 1
[junit4:junit4] 2> 109009 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[57 (1435450382777057280)]} 0 1
[junit4:junit4] 2> 109010 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[57 (1435450382777057280)]} 0 6
[junit4:junit4] 2> 109011 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[57]} 0 10
[junit4:junit4] 2> 109016 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[58 (1435450382787543040)]} 0 1
[junit4:junit4] 2> 109027 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[58 (1435450382795931648)]} 0 1
[junit4:junit4] 2> 109028 T446 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[58 (1435450382795931648)]} 0 6
[junit4:junit4] 2> 109029 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[58]} 0 10
[junit4:junit4] 2> 109033 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[59 (1435450382806417408)]} 0 1
[junit4:junit4] 2> 109044 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[59 (1435450382813757440)]} 0 1
[junit4:junit4] 2> 109045 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[59 (1435450382813757440)]} 0 6
[junit4:junit4] 2> 109046 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[59]} 0 10
[junit4:junit4] 2> 109050 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[60 (1435450382824243200)]} 0 1
[junit4:junit4] 2> 109057 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[60 (1435450382828437504)]} 0 1
[junit4:junit4] 2> 109058 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[60 (1435450382828437504)]} 0 5
[junit4:junit4] 2> 109062 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[61 (1435450382836826112)]} 0 1
[junit4:junit4] 2> 109073 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[61 (1435450382844166144)]} 0 1
[junit4:junit4] 2> 109074 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[61 (1435450382844166144)]} 0 5
[junit4:junit4] 2> 109075 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[61]} 0 10
[junit4:junit4] 2> 109079 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[62 (1435450382854651904)]} 0 1
[junit4:junit4] 2> 109086 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[62 (1435450382858846208)]} 0 1
[junit4:junit4] 2> 109087 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[62 (1435450382858846208)]} 0 5
[junit4:junit4] 2> 109091 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[63 (1435450382867234816)]} 0 1
[junit4:junit4] 2> 109101 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[63 (1435450382874574848)]} 0 0
[junit4:junit4] 2> 109102 T432 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[63 (1435450382874574848)]} 0 5
[junit4:junit4] 2> 109103 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[63]} 0 9
[junit4:junit4] 2> 109107 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[64 (1435450382884012032)]} 0 1
[junit4:junit4] 2> 109115 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[64 (1435450382888206336)]} 0 1
[junit4:junit4] 2> 109116 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[64 (1435450382888206336)]} 0 6
[junit4:junit4] 2> 109121 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[65 (1435450382898692096)]} 0 1
[junit4:junit4] 2> 109128 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[65 (1435450382902886400)]} 0 1
[junit4:junit4] 2> 109129 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[65 (1435450382902886400)]} 0 5
[junit4:junit4] 2> 109133 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[66 (1435450382911275008)]} 0 1
[junit4:junit4] 2> 109140 T458 C32 P43357 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43350/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[66 (1435450382915469312)]} 0 1
[junit4:junit4] 2> 109141 T430 C33 P43350 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[66 (1435450382915469312)]} 0 5
[junit4:junit4] 2> 109145 T406 C36 P43346 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[67 (1435450382923857920)]} 0 1
[junit4:junit4] 2> 109152 T477 C34 P43361 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={distrib.from=http://127.0.0.1:43353/zx/cp/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[67 (1435450382928052224)]} 0 1
[junit4:junit4] 2> 109153 T448 C35 P43353 oasup.LogUpdateProcessor.finish [collection1] webapp=/zx/cp path=/update params={wt=javabin&version=2} {add=[67 (1435450382928052224)]} 0 5
[ju
[...truncated too long message...]
l.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:43350/zx/cp returned non ok status:500, message:Server Error
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([B293A12E2D6924FD:33752F365A3644C1]:0)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
[junit4:junit4] > at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
[junit4:junit4] > at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:208)
[junit4:junit4] > at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:136)
[junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4:junit4] 2> 263576 T394 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4] 2> 288882 T485 oasu.PeerSync.handleResponse WARN PeerSync: core=collection1 url=http://127.0.0.1:43361/zx/cp couldn't connect to http://127.0.0.1:43353/zx/cp/collection1/, counting as success
[junit4:junit4] 2> 288883 T485 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:43361/zx/cp DONE. sync succeeded
[junit4:junit4] 2> 288883 T485 oasc.SyncStrategy.syncReplicas We have been closed, won't attempt to sync replicas back to leader
[junit4:junit4] 2> 288886 T485 oasc.ShardLeaderElectionContext.runLeaderProcess Sync was not a success but no one else is active! I am the leader
[junit4:junit4] 2> 288886 T485 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:43361/zx/cp/collection1/
[junit4:junit4] 2> 288886 T485 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@2c46465f
[junit4:junit4] 2> 288888 T485 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=4,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=623,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
[junit4:junit4] 2> 288889 T485 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
[junit4:junit4] 2> 288889 T485 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
[junit4:junit4] 2> 288889 T485 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
[junit4:junit4] 2> 288893 T485 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
[junit4:junit4] 2> 288893 T485 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
[junit4:junit4] 2> 288894 T485 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4;done=false>>]
[junit4:junit4] 2> 288894 T485 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4
[junit4:junit4] 2> 288894 T485 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index;done=false>>]
[junit4:junit4] 2> 288895 T485 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368952133840/jetty4/index
[junit4:junit4] 2> 288895 T485 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 288895 T485 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard2/election/89715647043731468-5-n_0000000001
[junit4:junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4:junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4:junit4] 2> at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
[junit4:junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:152)
[junit4:junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:149)
[junit4:junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:65)
[junit4:junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:149)
[junit4:junit4] 2> at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:63)
[junit4:junit4] 2> at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:260)
[junit4:junit4] 2> at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:156)
[junit4:junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:100)
[junit4:junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4:junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:129)
[junit4:junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4:junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4:junit4] 2>
[junit4:junit4] 2> 288896 T485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 288897 T485 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 288897 T485 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 288897 T485 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 288897 T485 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 288898 T485 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
[junit4:junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4:junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4:junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4:junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:235)
[junit4:junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:232)
[junit4:junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:65)
[junit4:junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:232)
[junit4:junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:84)
[junit4:junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4:junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:129)
[junit4:junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4:junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4:junit4] 2>
[junit4:junit4] 2> NOTE: test params are: codec=Lucene42: {id=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=Direct)}, docValues:{}, sim=DefaultSimilarity, locale=uk_UA, timezone=America/Argentina/Rio_Gallegos
[junit4:junit4] 2> NOTE: FreeBSD 9.0-RELEASE amd64/Oracle Corporation 1.7.0_17 (64-bit)/cpus=16,threads=2,free=112584056,total=214958080
[junit4:junit4] 2> NOTE: All tests run in this JVM: [TimeZoneUtilsTest, TestWriterPerf, TestBinaryField, SuggesterTSTTest, TestSolrQueryParserResource, TestShardHandlerFactory, QueryResultKeyTest, FileBasedSpellCheckerTest, SuggesterWFSTTest, TestWordDelimiterFilterFactory, TestOmitPositions, IndexBasedSpellCheckerTest, PluginInfoTest, TestSolrQueryParserDefaultOperatorResource, TestRecovery, TestLazyCores, URLClassifyProcessorTest, BadIndexSchemaTest, LeaderElectionTest, BinaryUpdateRequestHandlerTest, TestCopyFieldCollectionResource, ChaosMonkeyShardSplitTest]
[junit4:junit4] Completed on J0 in 209.34s, 1 test, 1 error <<< FAILURES!
[...truncated 759 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/build.xml:383: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/build.xml:363: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build.xml:181: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/common-build.xml:437: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/lucene/common-build.xml:1243: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/lucene/common-build.xml:887: There were test failures: 296 suites, 1234 tests, 1 error, 27 ignored (8 assumptions)
Total time: 49 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure
[JENKINS] Lucene-Solr-Tests-4.x-java7 - Build # 1261 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-java7/1261/
2 tests failed.
REGRESSION: org.apache.solr.cloud.SyncSliceTest.testDistribSearch
Error Message:
shard1 is not consistent. Got 305 from http://127.0.0.1:15616/i_/wu/collection1lastClient and got 5 from http://127.0.0.1:32544/i_/wu/collection1
Stack Trace:
java.lang.AssertionError: shard1 is not consistent. Got 305 from http://127.0.0.1:15616/i_/wu/collection1lastClient and got 5 from http://127.0.0.1:32544/i_/wu/collection1
at __randomizedtesting.SeedInfo.seed([6515B0E84615AFB1:E4F33EF0314ACF8D]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:963)
at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:238)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:722)
FAILED: org.apache.solr.cloud.ChaosMonkeyShardSplitTest.testDistribSearch
Error Message:
Server at http://127.0.0.1:36659/b_z/pe returned non ok status:500, message:Server Error
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:36659/b_z/pe returned non ok status:500, message:Server Error
at __randomizedtesting.SeedInfo.seed([6515B0E84615AFB1:E4F33EF0314ACF8D]:0)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
at org.apache.solr.cloud.ShardSplitTest.splitShard(ShardSplitTest.java:208)
at org.apache.solr.cloud.ChaosMonkeyShardSplitTest.doTest(ChaosMonkeyShardSplitTest.java:136)
at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:601)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:722)
Build Log:
[...truncated 9519 lines...]
[junit4:junit4] Suite: org.apache.solr.cloud.ChaosMonkeyShardSplitTest
[junit4:junit4] 2> 481792 T1245 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /b_z/pe
[junit4:junit4] 2> 481798 T1245 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
[junit4:junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./solrtest-ChaosMonkeyShardSplitTest-1368907268265
[junit4:junit4] 2> 481800 T1245 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4:junit4] 2> 481800 T1246 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4:junit4] 2> 481901 T1245 oasc.ZkTestServer.run start zk server on port:59305
[junit4:junit4] 2> 481902 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 481907 T1252 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25eaca50 name:ZooKeeperConnection Watcher:127.0.0.1:59305 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 481907 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 481907 T1245 oascc.SolrZkClient.makePath makePath: /solr
[junit4:junit4] 2> 481918 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 481919 T1254 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f68a02f name:ZooKeeperConnection Watcher:127.0.0.1:59305/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 481920 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 481920 T1245 oascc.SolrZkClient.makePath makePath: /collections/collection1
[junit4:junit4] 2> 481930 T1245 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
[junit4:junit4] 2> 481933 T1245 oascc.SolrZkClient.makePath makePath: /collections/control_collection
[junit4:junit4] 2> 481935 T1245 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
[junit4:junit4] 2> 481943 T1245 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 481944 T1245 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
[junit4:junit4] 2> 481957 T1245 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4:junit4] 2> 481957 T1245 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
[junit4:junit4] 2> 482061 T1245 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4:junit4] 2> 482062 T1245 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
[junit4:junit4] 2> 482065 T1245 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4:junit4] 2> 482066 T1245 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
[junit4:junit4] 2> 482069 T1245 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4:junit4] 2> 482069 T1245 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
[junit4:junit4] 2> 482072 T1245 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 482073 T1245 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
[junit4:junit4] 2> 482076 T1245 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 482077 T1245 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4:junit4] 2> 482080 T1245 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 482081 T1245 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
[junit4:junit4] 2> 482088 T1245 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4:junit4] 2> 482089 T1245 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
[junit4:junit4] 2> 482313 T1245 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 482321 T1245 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59308
[junit4:junit4] 2> 482322 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 482322 T1245 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 482323 T1245 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368907268566
[junit4:junit4] 2> 482323 T1245 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368907268566/solr.xml
[junit4:junit4] 2> 482324 T1245 oasc.CoreContainer.<init> New CoreContainer 780594142
[junit4:junit4] 2> 482324 T1245 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368907268566/'
[junit4:junit4] 2> 482325 T1245 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368907268566/'
[junit4:junit4] 2> 482441 T1245 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 482442 T1245 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 482442 T1245 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 482443 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 482443 T1245 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 482444 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 482444 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 482445 T1245 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 482445 T1245 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 482446 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 482462 T1245 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 482462 T1245 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59305/solr
[junit4:junit4] 2> 482463 T1245 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 482464 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 482466 T1265 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@24ce8073 name:ZooKeeperConnection Watcher:127.0.0.1:59305 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 482467 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 482469 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 482481 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 482483 T1267 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@79130d54 name:ZooKeeperConnection Watcher:127.0.0.1:59305/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 482483 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 482485 T1245 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4:junit4] 2> 482495 T1245 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4:junit4] 2> 482498 T1245 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4:junit4] 2> 482500 T1245 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59308_b_z%2Fpe
[junit4:junit4] 2> 482508 T1245 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59308_b_z%2Fpe
[junit4:junit4] 2> 482511 T1245 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4:junit4] 2> 482528 T1245 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4:junit4] 2> 482530 T1245 oasc.Overseer.start Overseer (id=89712706733342723-127.0.0.1:59308_b_z%2Fpe-n_0000000000) starting
[junit4:junit4] 2> 482539 T1245 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4:junit4] 2> 482549 T1269 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4:junit4] 2> 482549 T1245 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4:junit4] 2> 482552 T1245 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4:junit4] 2> 482559 T1245 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 482566 T1270 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368907268566/collection1
[junit4:junit4] 2> 482566 T1268 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4:junit4] 2> 482566 T1270 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
[junit4:junit4] 2> 482567 T1270 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 482567 T1270 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
[junit4:junit4] 2> 482568 T1270 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368907268566/collection1/'
[junit4:junit4] 2> 482569 T1270 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368907268566/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 482570 T1270 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368907268566/collection1/lib/README' to classloader
[junit4:junit4] 2> 482608 T1270 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 482652 T1270 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 482753 T1270 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 482758 T1270 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 483181 T1270 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 483184 T1270 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 483186 T1270 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 483190 T1270 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 483210 T1270 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 483210 T1270 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-controljetty-1368907268566/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/control/data/
[junit4:junit4] 2> 483211 T1270 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@facb7da
[junit4:junit4] 2> 483211 T1270 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 483212 T1270 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/control/data
[junit4:junit4] 2> 483212 T1270 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/control/data/index/
[junit4:junit4] 2> 483212 T1270 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/control/data/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 483212 T1270 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/control/data/index
[junit4:junit4] 2> 483215 T1270 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@8b5d458 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5e48fd92),segFN=segments_1,generation=1}
[junit4:junit4] 2> 483215 T1270 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 483217 T1270 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 483218 T1270 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 483218 T1270 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 483219 T1270 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 483219 T1270 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 483219 T1270 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 483220 T1270 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 483220 T1270 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 483220 T1270 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 483229 T1270 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 483234 T1270 oass.SolrIndexSearcher.<init> Opening Searcher@3e9bd0da main
[junit4:junit4] 2> 483234 T1270 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/control/data/tlog
[junit4:junit4] 2> 483235 T1270 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 483235 T1270 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 483238 T1271 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3e9bd0da main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 483240 T1270 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 483240 T1270 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 484071 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 484072 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:59308_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:59308/b_z/pe"}
[junit4:junit4] 2> 484072 T1268 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with numShards 1
[junit4:junit4] 2> 484072 T1268 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 484077 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 484243 T1270 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 484243 T1270 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59308/b_z/pe collection:control_collection shard:shard1
[junit4:junit4] 2> 484244 T1270 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
[junit4:junit4] 2> 484261 T1270 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 484272 T1270 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 484272 T1270 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 484273 T1270 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59308/b_z/pe/collection1/
[junit4:junit4] 2> 484273 T1270 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 484273 T1270 oasc.SyncStrategy.syncToMe http://127.0.0.1:59308/b_z/pe/collection1/ has no replicas
[junit4:junit4] 2> 484274 T1270 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59308/b_z/pe/collection1/
[junit4:junit4] 2> 484274 T1270 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
[junit4:junit4] 2> 485590 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 485613 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 485657 T1270 oasc.ZkController.register We are http://127.0.0.1:59308/b_z/pe/collection1/ and leader is http://127.0.0.1:59308/b_z/pe/collection1/
[junit4:junit4] 2> 485657 T1270 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59308/b_z/pe
[junit4:junit4] 2> 485657 T1270 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 485657 T1270 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 485658 T1270 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 485659 T1270 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 485660 T1245 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 485661 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 485662 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 485680 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:
[junit4:junit4] 2> 485682 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 485684 T1274 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@91e1a75 name:ZooKeeperConnection Watcher:127.0.0.1:59305/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 485685 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 485686 T1245 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 485688 T1245 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
[junit4:junit4] 2> 485943 T1245 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 485946 T1245 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:36659
[junit4:junit4] 2> 485947 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 485948 T1245 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 485948 T1245 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368907272155
[junit4:junit4] 2> 485949 T1245 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368907272155/solr.xml
[junit4:junit4] 2> 485949 T1245 oasc.CoreContainer.<init> New CoreContainer 367029617
[junit4:junit4] 2> 485950 T1245 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368907272155/'
[junit4:junit4] 2> 485950 T1245 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368907272155/'
[junit4:junit4] 2> 486083 T1245 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 486084 T1245 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 486085 T1245 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 486085 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 486086 T1245 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 486086 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 486087 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 486088 T1245 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 486088 T1245 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 486089 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 486105 T1245 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 486105 T1245 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59305/solr
[junit4:junit4] 2> 486106 T1245 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 486107 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 486110 T1285 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26c155cb name:ZooKeeperConnection Watcher:127.0.0.1:59305 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 486111 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 486126 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 486137 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 486140 T1287 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1dbf9dcf name:ZooKeeperConnection Watcher:127.0.0.1:59305/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 486140 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 486152 T1245 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 487118 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 487119 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"1",
[junit4:junit4] 2> "numShards":"1",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"control_collection",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:59308_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:59308/b_z/pe"}
[junit4:junit4] 2> 487136 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 487136 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 487136 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 487155 T1245 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36659_b_z%2Fpe
[junit4:junit4] 2> 487157 T1245 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36659_b_z%2Fpe
[junit4:junit4] 2> 487159 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 487159 T1274 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 487159 T1267 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 487160 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 487160 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 487160 T1287 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4:junit4] 2> 487173 T1288 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368907272155/collection1
[junit4:junit4] 2> 487173 T1288 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 487174 T1288 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 487174 T1288 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 487176 T1288 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368907272155/collection1/'
[junit4:junit4] 2> 487177 T1288 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368907272155/collection1/lib/README' to classloader
[junit4:junit4] 2> 487178 T1288 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368907272155/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 487235 T1288 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 487318 T1288 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 487419 T1288 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 487427 T1288 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 488038 T1288 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 488042 T1288 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 488045 T1288 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 488050 T1288 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 488070 T1288 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 488071 T1288 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty1-1368907272155/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty1/
[junit4:junit4] 2> 488071 T1288 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@facb7da
[junit4:junit4] 2> 488072 T1288 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 488073 T1288 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty1
[junit4:junit4] 2> 488073 T1288 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty1/index/
[junit4:junit4] 2> 488073 T1288 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty1/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 488074 T1288 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty1/index
[junit4:junit4] 2> 488078 T1288 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5c526d45 lockFactory=org.apache.lucene.store.NativeFSLockFactory@655e47f4),segFN=segments_1,generation=1}
[junit4:junit4] 2> 488078 T1288 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 488082 T1288 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 488082 T1288 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 488083 T1288 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 488084 T1288 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 488084 T1288 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 488085 T1288 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 488085 T1288 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 488086 T1288 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 488086 T1288 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 488100 T1288 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 488108 T1288 oass.SolrIndexSearcher.<init> Opening Searcher@16a64ecd main
[junit4:junit4] 2> 488108 T1288 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty1/tlog
[junit4:junit4] 2> 488109 T1288 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 488110 T1288 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 488114 T1289 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@16a64ecd main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 488116 T1288 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 488117 T1288 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 488641 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 488642 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:36659_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:36659/b_z/pe"}
[junit4:junit4] 2> 488642 T1268 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with numShards 2
[junit4:junit4] 2> 488642 T1268 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 488662 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 488662 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 488662 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 489119 T1288 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 489119 T1288 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:36659/b_z/pe collection:collection1 shard:shard1
[junit4:junit4] 2> 489120 T1288 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4:junit4] 2> 489138 T1288 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 489147 T1288 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 489147 T1288 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 489148 T1288 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:36659/b_z/pe/collection1/
[junit4:junit4] 2> 489148 T1288 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 489148 T1288 oasc.SyncStrategy.syncToMe http://127.0.0.1:36659/b_z/pe/collection1/ has no replicas
[junit4:junit4] 2> 489149 T1288 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:36659/b_z/pe/collection1/
[junit4:junit4] 2> 489149 T1288 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4:junit4] 2> 490166 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 490218 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 490218 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 490218 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 490232 T1288 oasc.ZkController.register We are http://127.0.0.1:36659/b_z/pe/collection1/ and leader is http://127.0.0.1:36659/b_z/pe/collection1/
[junit4:junit4] 2> 490232 T1288 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36659/b_z/pe
[junit4:junit4] 2> 490232 T1288 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 490232 T1288 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 490232 T1288 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 490234 T1288 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 490235 T1245 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 490235 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 490236 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 490412 T1245 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 490415 T1245 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:27067
[junit4:junit4] 2> 490416 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 490416 T1245 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 490417 T1245 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368907276714
[junit4:junit4] 2> 490417 T1245 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368907276714/solr.xml
[junit4:junit4] 2> 490417 T1245 oasc.CoreContainer.<init> New CoreContainer 193042597
[junit4:junit4] 2> 490418 T1245 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368907276714/'
[junit4:junit4] 2> 490418 T1245 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368907276714/'
[junit4:junit4] 2> 490510 T1245 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 490511 T1245 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 490512 T1245 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 490512 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 490512 T1245 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 490513 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 490513 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 490513 T1245 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 490514 T1245 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 490514 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 490526 T1245 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 490527 T1245 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59305/solr
[junit4:junit4] 2> 490527 T1245 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 490528 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 490531 T1301 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@249642ee name:ZooKeeperConnection Watcher:127.0.0.1:59305 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 490532 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 490556 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 490566 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 490568 T1303 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a52a7f name:ZooKeeperConnection Watcher:127.0.0.1:59305/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 490568 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 490582 T1245 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 491585 T1245 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:27067_b_z%2Fpe
[junit4:junit4] 2> 491588 T1245 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:27067_b_z%2Fpe
[junit4:junit4] 2> 491591 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 491591 T1267 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 491591 T1274 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 491591 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 491591 T1303 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 491592 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 491593 T1287 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4:junit4] 2> 491615 T1304 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368907276714/collection1
[junit4:junit4] 2> 491615 T1304 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 491616 T1304 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 491616 T1304 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 491618 T1304 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368907276714/collection1/'
[junit4:junit4] 2> 491619 T1304 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368907276714/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 491620 T1304 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368907276714/collection1/lib/README' to classloader
[junit4:junit4] 2> 491678 T1304 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 491731 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 491732 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"2",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:36659_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:36659/b_z/pe"}
[junit4:junit4] 2> 491738 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 491738 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 491738 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 491738 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 491765 T1304 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 491867 T1304 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 491874 T1304 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 492502 T1304 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 492506 T1304 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 492509 T1304 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 492514 T1304 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 492539 T1304 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 492539 T1304 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty2-1368907276714/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty2/
[junit4:junit4] 2> 492539 T1304 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@facb7da
[junit4:junit4] 2> 492540 T1304 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 492542 T1304 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty2
[junit4:junit4] 2> 492542 T1304 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty2/index/
[junit4:junit4] 2> 492542 T1304 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty2/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 492543 T1304 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty2/index
[junit4:junit4] 2> 492546 T1304 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7cd046e3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@66c0e9e6),segFN=segments_1,generation=1}
[junit4:junit4] 2> 492547 T1304 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 492550 T1304 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 492550 T1304 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 492551 T1304 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 492552 T1304 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 492552 T1304 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 492553 T1304 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 492553 T1304 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 492554 T1304 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 492554 T1304 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 492569 T1304 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 492576 T1304 oass.SolrIndexSearcher.<init> Opening Searcher@4acfc079 main
[junit4:junit4] 2> 492577 T1304 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty2/tlog
[junit4:junit4] 2> 492578 T1304 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 492578 T1304 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 492583 T1305 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4acfc079 main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 492585 T1304 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 492586 T1304 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 493249 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 493250 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:27067_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:27067/b_z/pe"}
[junit4:junit4] 2> 493251 T1268 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 493251 T1268 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 493266 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 493266 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 493266 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 493266 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 493589 T1304 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 493589 T1304 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:27067/b_z/pe collection:collection1 shard:shard2
[junit4:junit4] 2> 493590 T1304 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4:junit4] 2> 493609 T1304 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process.
[junit4:junit4] 2> 493620 T1304 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
[junit4:junit4] 2> 493620 T1304 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
[junit4:junit4] 2> 493620 T1304 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:27067/b_z/pe/collection1/
[junit4:junit4] 2> 493621 T1304 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
[junit4:junit4] 2> 493621 T1304 oasc.SyncStrategy.syncToMe http://127.0.0.1:27067/b_z/pe/collection1/ has no replicas
[junit4:junit4] 2> 493621 T1304 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:27067/b_z/pe/collection1/
[junit4:junit4] 2> 493621 T1304 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4:junit4] 2> 494770 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 494907 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 494907 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 494907 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 494907 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 494910 T1304 oasc.ZkController.register We are http://127.0.0.1:27067/b_z/pe/collection1/ and leader is http://127.0.0.1:27067/b_z/pe/collection1/
[junit4:junit4] 2> 494910 T1304 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:27067/b_z/pe
[junit4:junit4] 2> 494910 T1304 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
[junit4:junit4] 2> 494910 T1304 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 494910 T1304 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 494966 T1304 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 494968 T1245 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 494969 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 494970 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 495223 T1245 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 495227 T1245 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51545
[junit4:junit4] 2> 495227 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 495228 T1245 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 495228 T1245 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368907281465
[junit4:junit4] 2> 495228 T1245 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368907281465/solr.xml
[junit4:junit4] 2> 495229 T1245 oasc.CoreContainer.<init> New CoreContainer 1933298619
[junit4:junit4] 2> 495229 T1245 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368907281465/'
[junit4:junit4] 2> 495230 T1245 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368907281465/'
[junit4:junit4] 2> 495319 T1245 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 495320 T1245 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 495320 T1245 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 495321 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 495321 T1245 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 495321 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 495322 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 495322 T1245 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 495322 T1245 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 495323 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 495334 T1245 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 495335 T1245 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59305/solr
[junit4:junit4] 2> 495335 T1245 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 495336 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 495339 T1317 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2989336d name:ZooKeeperConnection Watcher:127.0.0.1:59305 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 495340 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 495357 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 495381 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 495388 T1319 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45e6db18 name:ZooKeeperConnection Watcher:127.0.0.1:59305/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 495388 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 495393 T1245 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 496397 T1245 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51545_b_z%2Fpe
[junit4:junit4] 2> 496427 T1245 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51545_b_z%2Fpe
[junit4:junit4] 2> 496431 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 496431 T1274 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 496431 T1319 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 496431 T1267 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 496431 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
[junit4:junit4] 2> 496432 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 496432 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 496433 T1287 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 496442 T1303 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4:junit4] 2> 496447 T1320 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368907281465/collection1
[junit4:junit4] 2> 496448 T1320 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 496448 T1320 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 496448 T1320 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 496449 T1320 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368907281465/collection1/'
[junit4:junit4] 2> 496450 T1320 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368907281465/collection1/lib/README' to classloader
[junit4:junit4] 2> 496451 T1320 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368907281465/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 496468 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 496468 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"3",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:27067_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:27067/b_z/pe"}
[junit4:junit4] 2> 496481 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 496482 T1319 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 496481 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 496481 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 496481 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 496489 T1320 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 496533 T1320 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 496634 T1320 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 496639 T1320 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 497052 T1320 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 497055 T1320 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 497057 T1320 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 497061 T1320 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 497079 T1320 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 497080 T1320 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty3-1368907281465/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/
[junit4:junit4] 2> 497080 T1320 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@facb7da
[junit4:junit4] 2> 497081 T1320 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 497081 T1320 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3
[junit4:junit4] 2> 497081 T1320 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/index/
[junit4:junit4] 2> 497082 T1320 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 497083 T1320 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/index
[junit4:junit4] 2> 497085 T1320 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1055cc82 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7b677c4c),segFN=segments_1,generation=1}
[junit4:junit4] 2> 497086 T1320 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 497088 T1320 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 497088 T1320 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 497089 T1320 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 497089 T1320 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 497090 T1320 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 497090 T1320 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 497090 T1320 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 497091 T1320 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 497091 T1320 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 497102 T1320 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 497107 T1320 oass.SolrIndexSearcher.<init> Opening Searcher@54abb1fa main
[junit4:junit4] 2> 497107 T1320 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/tlog
[junit4:junit4] 2> 497108 T1320 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 497108 T1320 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 497111 T1321 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@54abb1fa main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 497113 T1320 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 497113 T1320 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 497986 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 497987 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:51545_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:51545/b_z/pe"}
[junit4:junit4] 2> 497988 T1268 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 497988 T1268 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard1
[junit4:junit4] 2> 498001 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 498001 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 498001 T1319 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 498001 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 498001 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 498116 T1320 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 498116 T1320 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51545/b_z/pe collection:collection1 shard:shard1
[junit4:junit4] 2> 498120 T1320 oasc.ZkController.register We are http://127.0.0.1:51545/b_z/pe/collection1/ and leader is http://127.0.0.1:36659/b_z/pe/collection1/
[junit4:junit4] 2> 498121 T1320 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51545/b_z/pe
[junit4:junit4] 2> 498121 T1320 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 498121 T1320 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C127 name=collection1 org.apache.solr.core.SolrCore@55b4a934 url=http://127.0.0.1:51545/b_z/pe/collection1 node=127.0.0.1:51545_b_z%2Fpe C127_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:51545_b_z%2Fpe, base_url=http://127.0.0.1:51545/b_z/pe}
[junit4:junit4] 2> 498122 T1322 C127 P51545 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 498122 T1320 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 498123 T1322 C127 P51545 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 498123 T1322 C127 P51545 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 498123 T1322 C127 P51545 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 498124 T1245 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 498125 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 498125 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 498139 T1322 C127 P51545 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 498153 T1277 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 498390 T1245 oejs.Server.doStart jetty-8.1.10.v20130312
[junit4:junit4] 2> 498393 T1245 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:40620
[junit4:junit4] 2> 498394 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
[junit4:junit4] 2> 498395 T1245 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
[junit4:junit4] 2> 498395 T1245 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368907284610
[junit4:junit4] 2> 498396 T1245 oasc.CoreContainer$Initializer.initialize looking for solr config file: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368907284610/solr.xml
[junit4:junit4] 2> 498396 T1245 oasc.CoreContainer.<init> New CoreContainer 1549831729
[junit4:junit4] 2> 498397 T1245 oasc.CoreContainer.load Loading CoreContainer using Solr Home: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368907284610/'
[junit4:junit4] 2> 498398 T1245 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368907284610/'
[junit4:junit4] 2> 498547 T1245 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
[junit4:junit4] 2> 498548 T1245 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
[junit4:junit4] 2> 498548 T1245 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
[junit4:junit4] 2> 498549 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
[junit4:junit4] 2> 498549 T1245 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
[junit4:junit4] 2> 498550 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
[junit4:junit4] 2> 498550 T1245 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
[junit4:junit4] 2> 498551 T1245 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
[junit4:junit4] 2> 498551 T1245 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
[junit4:junit4] 2> 498552 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
[junit4:junit4] 2> 498568 T1245 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener
[junit4:junit4] 2> 498569 T1245 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59305/solr
[junit4:junit4] 2> 498569 T1245 oasc.ZkController.checkChrootPath zkHost includes chroot
[junit4:junit4] 2> 498571 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 498573 T1334 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1eccfd2c name:ZooKeeperConnection Watcher:127.0.0.1:59305 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 498574 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 498584 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
[junit4:junit4] 2> 498595 T1245 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4:junit4] 2> 498597 T1336 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f71af26 name:ZooKeeperConnection Watcher:127.0.0.1:59305/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4:junit4] 2> 498598 T1245 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4:junit4] 2> 498607 T1245 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4:junit4] 2> 499506 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 499507 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:51545_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:51545/b_z/pe"}
[junit4:junit4] 2> 499520 T1336 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 499521 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 499520 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 499520 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 499520 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 499520 T1319 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 499610 T1245 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40620_b_z%2Fpe
[junit4:junit4] 2> 499644 T1245 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40620_b_z%2Fpe
[junit4:junit4] 2> 499649 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 499649 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
[junit4:junit4] 2> 499649 T1274 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 499649 T1319 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 499649 T1336 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 499649 T1267 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 499650 T1336 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 499650 T1319 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 499650 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 499650 T1303 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 499650 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 499680 T1287 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4:junit4] 2> 499686 T1337 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368907284610/collection1
[junit4:junit4] 2> 499687 T1337 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
[junit4:junit4] 2> 499688 T1337 oasc.ZkController.createCollectionZkNode Collection zkNode exists
[junit4:junit4] 2> 499688 T1337 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
[junit4:junit4] 2> 499689 T1337 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368907284610/collection1/'
[junit4:junit4] 2> 499690 T1337 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368907284610/collection1/lib/README' to classloader
[junit4:junit4] 2> 499690 T1337 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368907284610/collection1/lib/classes/' to classloader
[junit4:junit4] 2> 499730 T1337 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_44
[junit4:junit4] 2> 499774 T1337 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
[junit4:junit4] 2> 499875 T1337 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
[junit4:junit4] 2> 499880 T1337 oass.IndexSchema.readSchema [collection1] Schema name=test
[junit4:junit4] 2> 500156 T1277 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 4, state: recovering, checkLive: true, onlyIfLeader: true for: 2 seconds.
[junit4:junit4] 2> 500156 T1277 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=4&state=recovering&nodeName=127.0.0.1:51545_b_z%252Fpe&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=2003
[junit4:junit4] 2> 500298 T1337 oass.IndexSchema.readSchema default search field in schema is text
[junit4:junit4] 2> 500301 T1337 oass.IndexSchema.readSchema unique key field: id
[junit4:junit4] 2> 500303 T1337 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 500307 T1337 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
[junit4:junit4] 2> 500326 T1337 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 500326 T1337 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-jetty4-1368907284610/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/
[junit4:junit4] 2> 500327 T1337 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@facb7da
[junit4:junit4] 2> 500327 T1337 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
[junit4:junit4] 2> 500328 T1337 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4
[junit4:junit4] 2> 500328 T1337 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/index/
[junit4:junit4] 2> 500328 T1337 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/index' doesn't exist. Creating new index...
[junit4:junit4] 2> 500329 T1337 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/index
[junit4:junit4] 2> 500331 T1337 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@93c7913 lockFactory=org.apache.lucene.store.NativeFSLockFactory@292b00d7),segFN=segments_1,generation=1}
[junit4:junit4] 2> 500331 T1337 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 500334 T1337 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
[junit4:junit4] 2> 500334 T1337 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
[junit4:junit4] 2> 500335 T1337 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
[junit4:junit4] 2> 500335 T1337 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
[junit4:junit4] 2> 500336 T1337 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
[junit4:junit4] 2> 500336 T1337 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
[junit4:junit4] 2> 500336 T1337 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
[junit4:junit4] 2> 500337 T1337 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
[junit4:junit4] 2> 500337 T1337 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
[junit4:junit4] 2> 500346 T1337 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
[junit4:junit4] 2> 500350 T1337 oass.SolrIndexSearcher.<init> Opening Searcher@786312e main
[junit4:junit4] 2> 500351 T1337 oasu.UpdateHandler.clearLog Clearing tlog files, tlogDir=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/tlog
[junit4:junit4] 2> 500352 T1337 oasu.CommitTracker.<init> Hard AutoCommit: disabled
[junit4:junit4] 2> 500352 T1337 oasu.CommitTracker.<init> Soft AutoCommit: disabled
[junit4:junit4] 2> 500356 T1338 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@786312e main{StandardDirectoryReader(segments_1:1)}
[junit4:junit4] 2> 500358 T1337 oasc.ZkController.publish publishing core=collection1 state=down
[junit4:junit4] 2> 500358 T1337 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 501025 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 501026 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":null,
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"down",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:40620_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:40620/b_z/pe"}
[junit4:junit4] 2> 501026 T1268 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
[junit4:junit4] 2> 501026 T1268 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard=shard2
[junit4:junit4] 2> 501068 T1319 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 501068 T1336 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 501068 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 501068 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 501068 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 501068 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 501361 T1337 oasc.CoreContainer.registerCore registering core: collection1
[junit4:junit4] 2> 501361 T1337 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:40620/b_z/pe collection:collection1 shard:shard2
[junit4:junit4] 2> 501364 T1337 oasc.ZkController.register We are http://127.0.0.1:40620/b_z/pe/collection1/ and leader is http://127.0.0.1:27067/b_z/pe/collection1/
[junit4:junit4] 2> 501364 T1337 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:40620/b_z/pe
[junit4:junit4] 2> 501365 T1337 oasc.ZkController.checkRecovery Core needs to recover:collection1
[junit4:junit4] 2> 501365 T1337 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
[junit4:junit4] 2> ASYNC NEW_CORE C128 name=collection1 org.apache.solr.core.SolrCore@5bedfb57 url=http://127.0.0.1:40620/b_z/pe/collection1 node=127.0.0.1:40620_b_z%2Fpe C128_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, collection=collection1, node_name=127.0.0.1:40620_b_z%2Fpe, base_url=http://127.0.0.1:40620/b_z/pe}
[junit4:junit4] 2> 501365 T1339 C128 P40620 oasc.RecoveryStrategy.run Starting recovery process. core=collection1 recoveringAfterStartup=true
[junit4:junit4] 2> 501366 T1337 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 501366 T1339 C128 P40620 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
[junit4:junit4] 2> 501366 T1339 C128 P40620 oasc.ZkController.publish publishing core=collection1 state=recovering
[junit4:junit4] 2> 501366 T1339 C128 P40620 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 501367 T1245 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0
[junit4:junit4] 2> 501368 T1245 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
[junit4:junit4] 2> 501368 T1245 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 501369 T1339 C128 P40620 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 501386 T1245 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 501386 T1293 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true
[junit4:junit4] 2> 501387 T1245 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 15 for each attempt
[junit4:junit4] 2> 501388 T1245 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):15
[junit4:junit4] 2> 501389 T1245 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> C127_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:51545_b_z%2Fpe, base_url=http://127.0.0.1:51545/b_z/pe}
[junit4:junit4] 2> 502158 T1322 C127 P51545 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:36659/b_z/pe/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 502158 T1322 C127 P51545 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:51545/b_z/pe START replicas=[http://127.0.0.1:36659/b_z/pe/collection1/] nUpdates=100
[junit4:junit4] 2> 502159 T1322 C127 P51545 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 502160 T1322 C127 P51545 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 502160 T1322 C127 P51545 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 502160 T1322 C127 P51545 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 502160 T1322 C127 P51545 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 502161 T1322 C127 P51545 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:36659/b_z/pe/collection1/. core=collection1
[junit4:junit4] 2> 502161 T1322 C127 P51545 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> ASYNC NEW_CORE C129 name=collection1 org.apache.solr.core.SolrCore@127ca7a url=http://127.0.0.1:36659/b_z/pe/collection1 node=127.0.0.1:36659_b_z%2Fpe C129_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:36659_b_z%2Fpe, base_url=http://127.0.0.1:36659/b_z/pe, leader=true}
[junit4:junit4] 2> 502166 T1278 C129 P36659 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 502175 T1279 C129 P36659 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 502176 T1279 C129 P36659 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5c526d45 lockFactory=org.apache.lucene.store.NativeFSLockFactory@655e47f4),segFN=segments_1,generation=1}
[junit4:junit4] 2> 502177 T1279 C129 P36659 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 502177 T1279 C129 P36659 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5c526d45 lockFactory=org.apache.lucene.store.NativeFSLockFactory@655e47f4),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5c526d45 lockFactory=org.apache.lucene.store.NativeFSLockFactory@655e47f4),segFN=segments_2,generation=2}
[junit4:junit4] 2> 502178 T1279 C129 P36659 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 502178 T1279 C129 P36659 oass.SolrIndexSearcher.<init> Opening Searcher@1f8352ad realtime
[junit4:junit4] 2> 502178 T1279 C129 P36659 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 502179 T1279 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 4
[junit4:junit4] 2> 502180 T1322 C127 P51545 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 502180 T1322 C127 P51545 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 502183 T1280 C129 P36659 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 502183 T1280 C129 P36659 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=2
[junit4:junit4] 2> 502184 T1322 C127 P51545 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 502184 T1322 C127 P51545 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 502184 T1322 C127 P51545 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 502187 T1280 C129 P36659 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 502188 T1322 C127 P51545 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 502188 T1322 C127 P51545 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/index.20130518130128654
[junit4:junit4] 2> 502189 T1322 C127 P51545 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@48445dc9 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5240850c) fullCopy=false
[junit4:junit4] 2> 502191 T1280 C129 P36659 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 502192 T1322 C127 P51545 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 502193 T1322 C127 P51545 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 502193 T1322 C127 P51545 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 502195 T1322 C127 P51545 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1055cc82 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7b677c4c),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1055cc82 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7b677c4c),segFN=segments_2,generation=2}
[junit4:junit4] 2> 502196 T1322 C127 P51545 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 502196 T1322 C127 P51545 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 502196 T1322 C127 P51545 oass.SolrIndexSearcher.<init> Opening Searcher@1658d78a main
[junit4:junit4] 2> 502197 T1321 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1658d78a main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 502198 T1322 C127 P51545 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/index.20130518130128654 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/index.20130518130128654;done=true>>]
[junit4:junit4] 2> 502198 T1322 C127 P51545 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/index.20130518130128654
[junit4:junit4] 2> 502198 T1322 C127 P51545 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty3/index.20130518130128654
[junit4:junit4] 2> 502199 T1322 C127 P51545 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 502199 T1322 C127 P51545 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 502199 T1322 C127 P51545 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 502199 T1322 C127 P51545 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 502226 T1322 C127 P51545 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 502390 T1245 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 502573 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 502574 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"recovering",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:40620_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:40620/b_z/pe"}
[junit4:junit4] 2> 502578 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"4",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard1",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:51545_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:51545/b_z/pe"}
[junit4:junit4] 2> 502596 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 502596 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 502596 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 502596 T1336 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 502596 T1319 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 502596 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 503388 T1293 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: 5, state: recovering, checkLive: true, onlyIfLeader: true for: 2 seconds.
[junit4:junit4] 2> 503388 T1293 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=5&state=recovering&nodeName=127.0.0.1:40620_b_z%252Fpe&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=2002
[junit4:junit4] 2> 503392 T1245 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 504394 T1245 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> C128_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=recovering, core=collection1, collection=collection1, node_name=127.0.0.1:40620_b_z%2Fpe, base_url=http://127.0.0.1:40620/b_z/pe}
[junit4:junit4] 2> 505390 T1339 C128 P40620 oasc.RecoveryStrategy.doRecovery Attempting to PeerSync from http://127.0.0.1:27067/b_z/pe/collection1/ core=collection1 - recoveringAfterStartup=true
[junit4:junit4] 2> 505390 T1339 C128 P40620 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:40620/b_z/pe START replicas=[http://127.0.0.1:27067/b_z/pe/collection1/] nUpdates=100
[junit4:junit4] 2> 505391 T1339 C128 P40620 oasu.PeerSync.sync WARN no frame of reference to tell of we've missed updates
[junit4:junit4] 2> 505391 T1339 C128 P40620 oasc.RecoveryStrategy.doRecovery PeerSync Recovery was not successful - trying replication. core=collection1
[junit4:junit4] 2> 505391 T1339 C128 P40620 oasc.RecoveryStrategy.doRecovery Starting Replication Recovery. core=collection1
[junit4:junit4] 2> 505391 T1339 C128 P40620 oasc.RecoveryStrategy.doRecovery Begin buffering updates. core=collection1
[junit4:junit4] 2> 505391 T1339 C128 P40620 oasu.UpdateLog.bufferUpdates Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4:junit4] 2> 505391 T1339 C128 P40620 oasc.RecoveryStrategy.replicate Attempting to replicate from http://127.0.0.1:27067/b_z/pe/collection1/. core=collection1
[junit4:junit4] 2> 505392 T1339 C128 P40620 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
[junit4:junit4] 2> 505396 T1245 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> ASYNC NEW_CORE C130 name=collection1 org.apache.solr.core.SolrCore@3c18aea7 url=http://127.0.0.1:27067/b_z/pe/collection1 node=127.0.0.1:27067_b_z%2Fpe C130_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:27067_b_z%2Fpe, base_url=http://127.0.0.1:27067/b_z/pe, leader=true}
[junit4:junit4] 2> 505398 T1294 C130 P27067 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=0
[junit4:junit4] 2> 505402 T1295 C130 P27067 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 505404 T1295 C130 P27067 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7cd046e3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@66c0e9e6),segFN=segments_1,generation=1}
[junit4:junit4] 2> 505404 T1295 C130 P27067 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 505405 T1295 C130 P27067 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7cd046e3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@66c0e9e6),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7cd046e3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@66c0e9e6),segFN=segments_2,generation=2}
[junit4:junit4] 2> 505405 T1295 C130 P27067 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 505405 T1295 C130 P27067 oass.SolrIndexSearcher.<init> Opening Searcher@78bebd61 realtime
[junit4:junit4] 2> 505406 T1295 C130 P27067 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 505406 T1295 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={waitSearcher=true&openSearcher=false&commit=true&wt=javabin&commit_end_point=true&version=2&softCommit=false} {commit=} 0 4
[junit4:junit4] 2> 505407 T1339 C128 P40620 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 505407 T1339 C128 P40620 oash.SnapPuller.<init> No value set for 'pollInterval'. Timer Task not started.
[junit4:junit4] 2> 505409 T1296 C130 P27067 oash.ReplicationHandler.inform Commits will be reserved for 10000
[junit4:junit4] 2> 505409 T1296 C130 P27067 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/replication params={command=indexversion&qt=/replication&wt=javabin&version=2} status=0 QTime=1
[junit4:junit4] 2> 505409 T1339 C128 P40620 oash.SnapPuller.fetchLatestIndex Master's generation: 2
[junit4:junit4] 2> 505410 T1339 C128 P40620 oash.SnapPuller.fetchLatestIndex Slave's generation: 1
[junit4:junit4] 2> 505410 T1339 C128 P40620 oash.SnapPuller.fetchLatestIndex Starting replication process
[junit4:junit4] 2> 505412 T1296 C130 P27067 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/replication params={command=filelist&qt=/replication&wt=javabin&generation=2&version=2} status=0 QTime=0
[junit4:junit4] 2> 505413 T1339 C128 P40620 oash.SnapPuller.fetchLatestIndex Number of files in latest index in master: 1
[junit4:junit4] 2> 505414 T1339 C128 P40620 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/index.20130518130131879
[junit4:junit4] 2> 505414 T1339 C128 P40620 oash.SnapPuller.fetchLatestIndex Starting download to MockDirWrapper(org.apache.lucene.store.RAMDirectory@68febe87 lockFactory=org.apache.lucene.store.NativeFSLockFactory@49c8ed51) fullCopy=false
[junit4:junit4] 2> 505417 T1296 C130 P27067 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/replication params={file=segments_2&command=filecontent&checksum=true&wt=filestream&qt=/replication&generation=2} status=0 QTime=0
[junit4:junit4] 2> 505419 T1339 C128 P40620 oash.SnapPuller.fetchLatestIndex Total time taken for download : 0 secs
[junit4:junit4] 2> 505419 T1339 C128 P40620 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
[junit4:junit4] 2> 505419 T1339 C128 P40620 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
[junit4:junit4] 2> 505421 T1339 C128 P40620 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@93c7913 lockFactory=org.apache.lucene.store.NativeFSLockFactory@292b00d7),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@93c7913 lockFactory=org.apache.lucene.store.NativeFSLockFactory@292b00d7),segFN=segments_2,generation=2}
[junit4:junit4] 2> 505421 T1339 C128 P40620 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 505421 T1339 C128 P40620 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
[junit4:junit4] 2> 505422 T1339 C128 P40620 oass.SolrIndexSearcher.<init> Opening Searcher@784e8580 main
[junit4:junit4] 2> 505423 T1338 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@784e8580 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 505423 T1339 C128 P40620 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/index.20130518130131879 [CachedDir<<refCount=0;path=./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/index.20130518130131879;done=true>>]
[junit4:junit4] 2> 505423 T1339 C128 P40620 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/index.20130518130131879
[junit4:junit4] 2> 505424 T1339 C128 P40620 oasc.CachingDirectoryFactory.closeCacheValue Removing directory before core close: ./org.apache.solr.cloud.ChaosMonkeyShardSplitTest-1368907268264/jetty4/index.20130518130131879
[junit4:junit4] 2> 505424 T1339 C128 P40620 oasc.RecoveryStrategy.replay No replay needed. core=collection1
[junit4:junit4] 2> 505424 T1339 C128 P40620 oasc.RecoveryStrategy.doRecovery Replication Recovery was successful - registering as Active. core=collection1
[junit4:junit4] 2> 505424 T1339 C128 P40620 oasc.ZkController.publish publishing core=collection1 state=active
[junit4:junit4] 2> 505424 T1339 C128 P40620 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
[junit4:junit4] 2> 505426 T1339 C128 P40620 oasc.RecoveryStrategy.doRecovery Finished recovery process. core=collection1
[junit4:junit4] 2> 505603 T1268 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 505604 T1268 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
[junit4:junit4] 2> "operation":"state",
[junit4:junit4] 2> "core_node_name":"5",
[junit4:junit4] 2> "numShards":"2",
[junit4:junit4] 2> "shard":"shard2",
[junit4:junit4] 2> "roles":null,
[junit4:junit4] 2> "state":"active",
[junit4:junit4] 2> "shard_state":"active",
[junit4:junit4] 2> "core":"collection1",
[junit4:junit4] 2> "collection":"collection1",
[junit4:junit4] 2> "shard_range":null,
[junit4:junit4] 2> "node_name":"127.0.0.1:40620_b_z%2Fpe",
[junit4:junit4] 2> "base_url":"http://127.0.0.1:40620/b_z/pe"}
[junit4:junit4] 2> 505622 T1303 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 505622 T1287 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 505622 T1319 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 505622 T1274 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 505622 T1267 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 505622 T1336 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
[junit4:junit4] 2> 506398 T1245 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 506399 T1245 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Recoveries finished - collection: collection1
[junit4:junit4] 2> ASYNC NEW_CORE C131 name=collection1 org.apache.solr.core.SolrCore@49d77e6c url=http://127.0.0.1:59308/b_z/pe/collection1 node=127.0.0.1:59308_b_z%2Fpe C131_STATE=coll:control_collection core:collection1 props:{shard=shard1, state=active, core=collection1, collection=control_collection, node_name=127.0.0.1:59308_b_z%2Fpe, base_url=http://127.0.0.1:59308/b_z/pe, leader=true}
[junit4:junit4] 2> 506407 T1257 C131 P59308 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 506408 T1257 C131 P59308 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@8b5d458 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5e48fd92),segFN=segments_1,generation=1}
[junit4:junit4] 2> 506409 T1257 C131 P59308 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
[junit4:junit4] 2> 506409 T1257 C131 P59308 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@8b5d458 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5e48fd92),segFN=segments_1,generation=1}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@8b5d458 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5e48fd92),segFN=segments_2,generation=2}
[junit4:junit4] 2> 506410 T1257 C131 P59308 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
[junit4:junit4] 2> 506410 T1257 C131 P59308 oass.SolrIndexSearcher.<init> Opening Searcher@60275043 main
[junit4:junit4] 2> 506411 T1257 C131 P59308 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 506411 T1271 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@60275043 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 506414 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 8
[junit4:junit4] 2> 506428 T1281 C129 P36659 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 506430 T1281 C129 P36659 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5c526d45 lockFactory=org.apache.lucene.store.NativeFSLockFactory@655e47f4),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5c526d45 lockFactory=org.apache.lucene.store.NativeFSLockFactory@655e47f4),segFN=segments_3,generation=3}
[junit4:junit4] 2> 506430 T1281 C129 P36659 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 506430 T1281 C129 P36659 oass.SolrIndexSearcher.<init> Opening Searcher@54f06699 main
[junit4:junit4] 2> 506431 T1281 C129 P36659 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 506432 T1289 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@54f06699 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 506432 T1281 C129 P36659 oasu.SolrCmdDistributor.distribCommit Distrib commit to:[StdNode: http://127.0.0.1:51545/b_z/pe/collection1/, StdNode: http://127.0.0.1:27067/b_z/pe/collection1/, StdNode: http://127.0.0.1:40620/b_z/pe/collection1/] params:commit_end_point=true&commit=true&softCommit=false&waitSearcher=true&expungeDeletes=false
[junit4:junit4] 2> 506436 T1297 C130 P27067 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 506439 T1297 C130 P27067 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7cd046e3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@66c0e9e6),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7cd046e3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@66c0e9e6),segFN=segments_3,generation=3}
[junit4:junit4] 2> 506440 T1297 C130 P27067 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 506440 T1297 C130 P27067 oass.SolrIndexSearcher.<init> Opening Searcher@7d626162 main
[junit4:junit4] 2> 506441 T1297 C130 P27067 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 506442 T1305 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7d626162 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 506443 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 7
[junit4:junit4] 2> C127_STATE=coll:collection1 core:collection1 props:{shard=shard1, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:51545_b_z%2Fpe, base_url=http://127.0.0.1:51545/b_z/pe}
[junit4:junit4] 2> 506445 T1309 C127 P51545 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 506446 T1309 C127 P51545 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1055cc82 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7b677c4c),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1055cc82 lockFactory=org.apache.lucene.store.NativeFSLockFactory@7b677c4c),segFN=segments_3,generation=3}
[junit4:junit4] 2> 506446 T1309 C127 P51545 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> C128_STATE=coll:collection1 core:collection1 props:{shard=shard2, state=active, core=collection1, collection=collection1, node_name=127.0.0.1:40620_b_z%2Fpe, base_url=http://127.0.0.1:40620/b_z/pe}
[junit4:junit4] 2> 506446 T1326 C128 P40620 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4:junit4] 2> 506447 T1309 C127 P51545 oass.SolrIndexSearcher.<init> Opening Searcher@11d2cb25 main
[junit4:junit4] 2> 506448 T1326 C128 P40620 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@93c7913 lockFactory=org.apache.lucene.store.NativeFSLockFactory@292b00d7),segFN=segments_2,generation=2}
[junit4:junit4] 2> commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@93c7913 lockFactory=org.apache.lucene.store.NativeFSLockFactory@292b00d7),segFN=segments_3,generation=3}
[junit4:junit4] 2> 506448 T1309 C127 P51545 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 506449 T1321 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@11d2cb25 main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 506448 T1326 C128 P40620 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
[junit4:junit4] 2> 506449 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 4
[junit4:junit4] 2> 506450 T1326 C128 P40620 oass.SolrIndexSearcher.<init> Opening Searcher@ef07dab main
[junit4:junit4] 2> 506451 T1326 C128 P40620 oasu.DirectUpdateHandler2.commit end_commit_flush
[junit4:junit4] 2> 506452 T1338 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ef07dab main{StandardDirectoryReader(segments_2:1:nrt)}
[junit4:junit4] 2> 506452 T1326 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={waitSearcher=true&commit=true&wt=javabin&expungeDeletes=false&commit_end_point=true&version=2&softCommit=false} {commit=} 0 6
[junit4:junit4] 2> 506453 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={waitSearcher=true&commit=true&wt=javabin&version=2&softCommit=false} {commit=} 0 25
[junit4:junit4] 2> 506454 T1245 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4:junit4] 2> 506457 T1282 C129 P36659 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 506460 T1310 C127 P51545 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 506462 T1298 C130 P27067 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=0
[junit4:junit4] 2> 506465 T1327 C128 P40620 oasc.SolrCore.execute [collection1] webapp=/b_z/pe path=/select params={tests=checkShardConsistency&q=*:*&distrib=false&wt=javabin&version=2&rows=0} hits=0 status=0 QTime=1
[junit4:junit4] 2> 508470 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1435403335693762560)} 0 2
[junit4:junit4] 2> 508478 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={update.distrib=FROMLEADER&_version_=-1435403335697956864&update.from=http://127.0.0.1:27067/b_z/pe/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1435403335697956864)} 0 1
[junit4:junit4] 2> 508482 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={update.distrib=FROMLEADER&_version_=-1435403335702151168&update.from=http://127.0.0.1:36659/b_z/pe/collection1/&wt=javabin&version=2} {deleteByQuery=*:* (-1435403335702151168)} 0 1
[junit4:junit4] 2> 508483 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={update.distrib=TOLEADER&wt=javabin&version=2} {deleteByQuery=*:* (-1435403335702151168)} 0 6
[junit4:junit4] 2> 508484 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1435403335697956864)} 0 11
[junit4:junit4] 2> 508488 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[0 (1435403335712636928)]} 0 1
[junit4:junit4] 2> 508497 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[0 (1435403335718928384)]} 0 1
[junit4:junit4] 2> 508498 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[0 (1435403335718928384)]} 0 5
[junit4:junit4] 2> 508498 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[0]} 0 7
[junit4:junit4] 2> 508503 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[1 (1435403335728365568)]} 0 1
[junit4:junit4] 2> 508510 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[1 (1435403335732559872)]} 0 1
[junit4:junit4] 2> 508511 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[1 (1435403335732559872)]} 0 5
[junit4:junit4] 2> 508514 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[2 (1435403335739899904)]} 0 1
[junit4:junit4] 2> 508524 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[2 (1435403335747239936)]} 0 1
[junit4:junit4] 2> 508524 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[2 (1435403335747239936)]} 0 4
[junit4:junit4] 2> 508525 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[2]} 0 8
[junit4:junit4] 2> 508529 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[3 (1435403335755628544)]} 0 1
[junit4:junit4] 2> 508537 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[3 (1435403335761920000)]} 0 0
[junit4:junit4] 2> 508538 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[3 (1435403335761920000)]} 0 4
[junit4:junit4] 2> 508539 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[3]} 0 8
[junit4:junit4] 2> 508542 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[4 (1435403335770308608)]} 0 0
[junit4:junit4] 2> 508552 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[4 (1435403335776600064)]} 0 1
[junit4:junit4] 2> 508552 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[4 (1435403335776600064)]} 0 4
[junit4:junit4] 2> 508553 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[4]} 0 8
[junit4:junit4] 2> 508556 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[5 (1435403335784988672)]} 0 0
[junit4:junit4] 2> 508563 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[5 (1435403335788134400)]} 0 0
[junit4:junit4] 2> 508563 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[5 (1435403335788134400)]} 0 4
[junit4:junit4] 2> 508567 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[6 (1435403335795474432)]} 0 1
[junit4:junit4] 2> 508574 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[6 (1435403335799668736)]} 0 1
[junit4:junit4] 2> 508574 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[6 (1435403335799668736)]} 0 4
[junit4:junit4] 2> 508575 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[6]} 0 7
[junit4:junit4] 2> 508579 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[7 (1435403335808057344)]} 0 1
[junit4:junit4] 2> 508586 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[7 (1435403335812251648)]} 0 1
[junit4:junit4] 2> 508586 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[7 (1435403335812251648)]} 0 4
[junit4:junit4] 2> 508590 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[8 (1435403335819591680)]} 0 1
[junit4:junit4] 2> 508595 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[8 (1435403335822737408)]} 0 0
[junit4:junit4] 2> 508596 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[8 (1435403335822737408)]} 0 4
[junit4:junit4] 2> 508600 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[9 (1435403335830077440)]} 0 1
[junit4:junit4] 2> 508608 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[9 (1435403335836368896)]} 0 0
[junit4:junit4] 2> 508609 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[9 (1435403335836368896)]} 0 4
[junit4:junit4] 2> 508610 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[9]} 0 8
[junit4:junit4] 2> 508614 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[10 (1435403335844757504)]} 0 1
[junit4:junit4] 2> 508620 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[10 (1435403335847903232)]} 0 1
[junit4:junit4] 2> 508620 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[10 (1435403335847903232)]} 0 4
[junit4:junit4] 2> 508623 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[11 (1435403335855243264)]} 0 0
[junit4:junit4] 2> 508631 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[11 (1435403335859437568)]} 0 1
[junit4:junit4] 2> 508631 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[11 (1435403335859437568)]} 0 4
[junit4:junit4] 2> 508632 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[11]} 0 6
[junit4:junit4] 2> 508636 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[12 (1435403335867826176)]} 0 1
[junit4:junit4] 2> 508643 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[12 (1435403335873069056)]} 0 0
[junit4:junit4] 2> 508644 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[12 (1435403335873069056)]} 0 4
[junit4:junit4] 2> 508645 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[12]} 0 7
[junit4:junit4] 2> 508648 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[13 (1435403335881457664)]} 0 0
[junit4:junit4] 2> 508658 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[13 (1435403335887749120)]} 0 1
[junit4:junit4] 2> 508658 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[13 (1435403335887749120)]} 0 4
[junit4:junit4] 2> 508659 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[13]} 0 8
[junit4:junit4] 2> 508663 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[14 (1435403335896137728)]} 0 1
[junit4:junit4] 2> 508672 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[14 (1435403335902429184)]} 0 1
[junit4:junit4] 2> 508672 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[14 (1435403335902429184)]} 0 4
[junit4:junit4] 2> 508673 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[14]} 0 8
[junit4:junit4] 2> 508677 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[15 (1435403335910817792)]} 0 1
[junit4:junit4] 2> 508684 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[15 (1435403335915012096)]} 0 1
[junit4:junit4] 2> 508684 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[15 (1435403335915012096)]} 0 4
[junit4:junit4] 2> 508687 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[16 (1435403335922352128)]} 0 0
[junit4:junit4] 2> 508698 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[16 (1435403335929692160)]} 0 1
[junit4:junit4] 2> 508698 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[16 (1435403335929692160)]} 0 4
[junit4:junit4] 2> 508702 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[17 (1435403335937032192)]} 0 1
[junit4:junit4] 2> 508709 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[17 (1435403335941226496)]} 0 1
[junit4:junit4] 2> 508709 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[17 (1435403335941226496)]} 0 4
[junit4:junit4] 2> 508713 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[18 (1435403335948566528)]} 0 1
[junit4:junit4] 2> 508721 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[18 (1435403335953809408)]} 0 1
[junit4:junit4] 2> 508721 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[18 (1435403335953809408)]} 0 4
[junit4:junit4] 2> 508722 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[18]} 0 7
[junit4:junit4] 2> 508726 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[19 (1435403335962198016)]} 0 1
[junit4:junit4] 2> 508735 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[19 (1435403335968489472)]} 0 1
[junit4:junit4] 2> 508735 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[19 (1435403335968489472)]} 0 4
[junit4:junit4] 2> 508736 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[19]} 0 8
[junit4:junit4] 2> 508740 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[20 (1435403335976878080)]} 0 1
[junit4:junit4] 2> 508748 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[20 (1435403335982120960)]} 0 1
[junit4:junit4] 2> 508748 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[20 (1435403335982120960)]} 0 4
[junit4:junit4] 2> 508749 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[20]} 0 7
[junit4:junit4] 2> 508753 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[21 (1435403335990509568)]} 0 1
[junit4:junit4] 2> 508762 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[21 (1435403335996801024)]} 0 1
[junit4:junit4] 2> 508762 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[21 (1435403335996801024)]} 0 4
[junit4:junit4] 2> 508763 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[21]} 0 8
[junit4:junit4] 2> 508767 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[22 (1435403336005189632)]} 0 1
[junit4:junit4] 2> 508773 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[22 (1435403336008335360)]} 0 1
[junit4:junit4] 2> 508773 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[22 (1435403336008335360)]} 0 4
[junit4:junit4] 2> 508777 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[23 (1435403336015675392)]} 0 1
[junit4:junit4] 2> 508787 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[23 (1435403336023015424)]} 0 1
[junit4:junit4] 2> 508787 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[23 (1435403336023015424)]} 0 4
[junit4:junit4] 2> 508788 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[23]} 0 8
[junit4:junit4] 2> 508792 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[24 (1435403336031404032)]} 0 1
[junit4:junit4] 2> 508798 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[24 (1435403336034549760)]} 0 1
[junit4:junit4] 2> 508798 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[24 (1435403336034549760)]} 0 4
[junit4:junit4] 2> 508802 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[25 (1435403336041889792)]} 0 1
[junit4:junit4] 2> 508811 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[25 (1435403336048181248)]} 0 1
[junit4:junit4] 2> 508811 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[25 (1435403336048181248)]} 0 4
[junit4:junit4] 2> 508812 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[25]} 0 8
[junit4:junit4] 2> 508816 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[26 (1435403336056569856)]} 0 1
[junit4:junit4] 2> 508823 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[26 (1435403336060764160)]} 0 1
[junit4:junit4] 2> 508823 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[26 (1435403336060764160)]} 0 4
[junit4:junit4] 2> 508827 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[27 (1435403336068104192)]} 0 1
[junit4:junit4] 2> 508835 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[27 (1435403336073347072)]} 0 1
[junit4:junit4] 2> 508835 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[27 (1435403336073347072)]} 0 4
[junit4:junit4] 2> 508836 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[27]} 0 7
[junit4:junit4] 2> 508840 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[28 (1435403336081735680)]} 0 1
[junit4:junit4] 2> 508847 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[28 (1435403336085929984)]} 0 1
[junit4:junit4] 2> 508847 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[28 (1435403336085929984)]} 0 4
[junit4:junit4] 2> 508851 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[29 (1435403336093270016)]} 0 1
[junit4:junit4] 2> 508859 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[29 (1435403336099561472)]} 0 1
[junit4:junit4] 2> 508859 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[29 (1435403336099561472)]} 0 4
[junit4:junit4] 2> 508860 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[29]} 0 7
[junit4:junit4] 2> 508864 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[30 (1435403336106901504)]} 0 1
[junit4:junit4] 2> 508870 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[30 (1435403336110047232)]} 0 1
[junit4:junit4] 2> 508870 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[30 (1435403336110047232)]} 0 4
[junit4:junit4] 2> 508874 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[31 (1435403336117387264)]} 0 1
[junit4:junit4] 2> 508880 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[31 (1435403336120532992)]} 0 1
[junit4:junit4] 2> 508880 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[31 (1435403336120532992)]} 0 4
[junit4:junit4] 2> 508884 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[32 (1435403336127873024)]} 0 1
[junit4:junit4] 2> 508890 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[32 (1435403336131018752)]} 0 1
[junit4:junit4] 2> 508890 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[32 (1435403336131018752)]} 0 4
[junit4:junit4] 2> 508894 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[33 (1435403336138358784)]} 0 1
[junit4:junit4] 2> 508901 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[33 (1435403336142553088)]} 0 1
[junit4:junit4] 2> 508902 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[33 (1435403336142553088)]} 0 5
[junit4:junit4] 2> 508906 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[34 (1435403336150941696)]} 0 1
[junit4:junit4] 2> 508913 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[34 (1435403336155136000)]} 0 1
[junit4:junit4] 2> 508914 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[34 (1435403336155136000)]} 0 5
[junit4:junit4] 2> 508918 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[35 (1435403336163524608)]} 0 1
[junit4:junit4] 2> 508927 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[35 (1435403336169816064)]} 0 1
[junit4:junit4] 2> 508928 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[35 (1435403336169816064)]} 0 5
[junit4:junit4] 2> 508929 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[35]} 0 8
[junit4:junit4] 2> 508933 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[36 (1435403336179253248)]} 0 1
[junit4:junit4] 2> 508940 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[36 (1435403336183447552)]} 0 1
[junit4:junit4] 2> 508940 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[36 (1435403336183447552)]} 0 4
[junit4:junit4] 2> 508944 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[37 (1435403336190787584)]} 0 1
[junit4:junit4] 2> 508952 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[37 (1435403336196030464)]} 0 1
[junit4:junit4] 2> 508952 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[37 (1435403336196030464)]} 0 4
[junit4:junit4] 2> 508953 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[37]} 0 7
[junit4:junit4] 2> 508957 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[38 (1435403336204419072)]} 0 0
[junit4:junit4] 2> 508966 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[38 (1435403336211759104)]} 0 1
[junit4:junit4] 2> 508967 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[38 (1435403336211759104)]} 0 5
[junit4:junit4] 2> 508967 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[38]} 0 8
[junit4:junit4] 2> 508971 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[39 (1435403336219099136)]} 0 1
[junit4:junit4] 2> 508978 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[39 (1435403336223293440)]} 0 1
[junit4:junit4] 2> 508978 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[39 (1435403336223293440)]} 0 4
[junit4:junit4] 2> 508982 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[40 (1435403336230633472)]} 0 1
[junit4:junit4] 2> 508988 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[40 (1435403336233779200)]} 0 1
[junit4:junit4] 2> 508989 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[40 (1435403336233779200)]} 0 5
[junit4:junit4] 2> 508993 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[41 (1435403336242167808)]} 0 1
[junit4:junit4] 2> 508999 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[41 (1435403336245313536)]} 0 1
[junit4:junit4] 2> 508999 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[41 (1435403336245313536)]} 0 4
[junit4:junit4] 2> 509003 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[42 (1435403336252653568)]} 0 1
[junit4:junit4] 2> 509009 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[42 (1435403336255799296)]} 0 1
[junit4:junit4] 2> 509009 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[42 (1435403336255799296)]} 0 4
[junit4:junit4] 2> 509013 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[43 (1435403336263139328)]} 0 1
[junit4:junit4] 2> 509019 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[43 (1435403336266285056)]} 0 1
[junit4:junit4] 2> 509019 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[43 (1435403336266285056)]} 0 4
[junit4:junit4] 2> 509023 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[44 (1435403336273625088)]} 0 1
[junit4:junit4] 2> 509032 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[44 (1435403336279916544)]} 0 1
[junit4:junit4] 2> 509032 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[44 (1435403336279916544)]} 0 4
[junit4:junit4] 2> 509033 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[44]} 0 7
[junit4:junit4] 2> 509037 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[45 (1435403336288305152)]} 0 1
[junit4:junit4] 2> 509044 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[45 (1435403336292499456)]} 0 1
[junit4:junit4] 2> 509044 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[45 (1435403336292499456)]} 0 4
[junit4:junit4] 2> 509048 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[46 (1435403336299839488)]} 0 0
[junit4:junit4] 2> 509054 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[46 (1435403336302985216)]} 0 1
[junit4:junit4] 2> 509054 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[46 (1435403336302985216)]} 0 4
[junit4:junit4] 2> 509058 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[47 (1435403336310325248)]} 0 1
[junit4:junit4] 2> 509068 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[47 (1435403336317665280)]} 0 1
[junit4:junit4] 2> 509068 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[47 (1435403336317665280)]} 0 4
[junit4:junit4] 2> 509069 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[47]} 0 8
[junit4:junit4] 2> 509073 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[48 (1435403336326053888)]} 0 1
[junit4:junit4] 2> 509083 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[48 (1435403336333393920)]} 0 1
[junit4:junit4] 2> 509084 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[48 (1435403336333393920)]} 0 5
[junit4:junit4] 2> 509085 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[48]} 0 9
[junit4:junit4] 2> 509088 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[49 (1435403336341782528)]} 0 1
[junit4:junit4] 2> 509095 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[49 (1435403336347025408)]} 0 1
[junit4:junit4] 2> 509096 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[49 (1435403336347025408)]} 0 5
[junit4:junit4] 2> 509100 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[50 (1435403336354365440)]} 0 1
[junit4:junit4] 2> 509107 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[50 (1435403336358559744)]} 0 1
[junit4:junit4] 2> 509108 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[50 (1435403336358559744)]} 0 5
[junit4:junit4] 2> 509111 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[51 (1435403336365899776)]} 0 1
[junit4:junit4] 2> 509120 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[51 (1435403336372191232)]} 0 1
[junit4:junit4] 2> 509121 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[51 (1435403336372191232)]} 0 5
[junit4:junit4] 2> 509121 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[51]} 0 8
[junit4:junit4] 2> 509125 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[52 (1435403336380579840)]} 0 1
[junit4:junit4] 2> 509135 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[52 (1435403336387919872)]} 0 1
[junit4:junit4] 2> 509135 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[52 (1435403336387919872)]} 0 4
[junit4:junit4] 2> 509136 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[52]} 0 8
[junit4:junit4] 2> 509140 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[53 (1435403336396308480)]} 0 1
[junit4:junit4] 2> 509146 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[53 (1435403336399454208)]} 0 1
[junit4:junit4] 2> 509146 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[53 (1435403336399454208)]} 0 4
[junit4:junit4] 2> 509150 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[54 (1435403336406794240)]} 0 1
[junit4:junit4] 2> 509157 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[54 (1435403336410988544)]} 0 1
[junit4:junit4] 2> 509158 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[54 (1435403336410988544)]} 0 5
[junit4:junit4] 2> 509161 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[55 (1435403336418328576)]} 0 1
[junit4:junit4] 2> 509169 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[55 (1435403336423571456)]} 0 1
[junit4:junit4] 2> 509170 T1283 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[55 (1435403336423571456)]} 0 5
[junit4:junit4] 2> 509171 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[55]} 0 8
[junit4:junit4] 2> 509174 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[56 (1435403336431960064)]} 0 1
[junit4:junit4] 2> 509181 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[56 (1435403336436154368)]} 0 1
[junit4:junit4] 2> 509181 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[56 (1435403336436154368)]} 0 4
[junit4:junit4] 2> 509185 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[57 (1435403336443494400)]} 0 1
[junit4:junit4] 2> 509194 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[57 (1435403336449785856)]} 0 1
[junit4:junit4] 2> 509194 T1297 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=TOLEADER&wt=javabin&version=2} {add=[57 (1435403336449785856)]} 0 4
[junit4:junit4] 2> 509195 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[57]} 0 8
[junit4:junit4] 2> 509199 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[58 (1435403336458174464)]} 0 1
[junit4:junit4] 2> 509205 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[58 (1435403336461320192)]} 0 0
[junit4:junit4] 2> 509205 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[58 (1435403336461320192)]} 0 4
[junit4:junit4] 2> 509208 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[59 (1435403336468660224)]} 0 0
[junit4:junit4] 2> 509215 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[59 (1435403336471805952)]} 0 1
[junit4:junit4] 2> 509215 T1281 C129 P36659 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[59 (1435403336471805952)]} 0 4
[junit4:junit4] 2> 509219 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[60 (1435403336479145984)]} 0 1
[junit4:junit4] 2> 509225 T1328 C128 P40620 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:27067/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[60 (1435403336482291712)]} 0 1
[junit4:junit4] 2> 509225 T1299 C130 P27067 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[60 (1435403336482291712)]} 0 4
[junit4:junit4] 2> 509229 T1257 C131 P59308 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={wt=javabin&version=2} {add=[61 (1435403336489631744)]} 0 1
[junit4:junit4] 2> 509237 T1309 C127 P51545 oasup.LogUpdateProcessor.finish [collection1] webapp=/b_z/pe path=/update params={distrib.from=http://127.0.0.1:36659/b_z/pe/collection1/&update.distrib=FROMLEADER&wt=javabin&version=2} {add=[61 (1435403336495923200)]} 0 0
[junit4:junit4] 2> 509238 T1283 C129 P36659 oasup.LogUpdateProcesso
[...truncated too long message...]
ions/control_collection/leaders/shard1 (0)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {
[junit4:junit4] 1> "core":"collection1",
[junit4:junit4] 1> "node_name":"127.0.0.1:15586_i_%2Fwu",
[junit4:junit4] 1> "base_url":"http://127.0.0.1:15586/i_/wu"}
[junit4:junit4] 1> /solr/overseer_elect (2)
[junit4:junit4] 1> /solr/overseer_elect/election (4)
[junit4:junit4] 1> /solr/overseer_elect/election/89712732922118147-127.0.0.1:15586_i_%2Fwu-n_0000000000 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89712732922118158-127.0.0.1:15594_i_%2Fwu-n_0000000005 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89712732922118156-127.0.0.1:32544_i_%2Fwu-n_0000000004 (0)
[junit4:junit4] 1> /solr/overseer_elect/election/89712732922118154-127.0.0.1:15616_i_%2Fwu-n_0000000003 (0)
[junit4:junit4] 1> /solr/overseer_elect/leader (0)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1> {"id":"89712732922118147-127.0.0.1:15586_i_%2Fwu-n_0000000000"}
[junit4:junit4] 1> /zookeeper (1)
[junit4:junit4] 1> DATA:
[junit4:junit4] 1>
[junit4:junit4] 1>
[junit4:junit4] 2> NOTE: reproduce with: ant test -Dtestcase=SyncSliceTest -Dtests.method=testDistribSearch -Dtests.seed=6515B0E84615AFB1 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=sr_CS -Dtests.timezone=EAT -Dtests.file.encoding=US-ASCII
[junit4:junit4] FAILURE 883s J0 | SyncSliceTest.testDistribSearch <<<
[junit4:junit4] > Throwable #1: java.lang.AssertionError: shard1 is not consistent. Got 305 from http://127.0.0.1:15616/i_/wu/collection1lastClient and got 5 from http://127.0.0.1:32544/i_/wu/collection1
[junit4:junit4] > at __randomizedtesting.SeedInfo.seed([6515B0E84615AFB1:E4F33EF0314ACF8D]:0)
[junit4:junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:963)
[junit4:junit4] > at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:238)
[junit4:junit4] > at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:815)
[junit4:junit4] > at java.lang.Thread.run(Thread.java:722)
[junit4:junit4] 2> 1764241 T1922 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4:junit4] 2> 1765005 T2016 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1765005 T2016 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
[junit4:junit4] 2> 1765005 T2016 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1791590 T1997 oasu.PeerSync.handleResponse WARN PeerSync: core=collection1 url=http://127.0.0.1:15616/i_/wu couldn't connect to http://127.0.0.1:15594/i_/wu/collection1/, counting as success
[junit4:junit4] 2> 1791590 T1997 oasu.PeerSync.sync PeerSync: core=collection1 url=http://127.0.0.1:15616/i_/wu DONE. sync succeeded
[junit4:junit4] 2> 1791591 T1997 oasc.SyncStrategy.syncReplicas We have been closed, won't attempt to sync replicas back to leader
[junit4:junit4] 2> 1791591 T1997 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
[junit4:junit4] 2> 1791592 T1997 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@632dac33
[junit4:junit4] 2> 1791593 T1997 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=51,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=5,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
[junit4:junit4] 2> 1791594 T1997 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
[junit4:junit4] 2> 1791594 T1997 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
[junit4:junit4] 2> 1791594 T1997 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
[junit4:junit4] 2> 1791597 T1997 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
[junit4:junit4] 2> 1791599 T1997 oasc.CachingDirectoryFactory.close Closing NRTCachingDirectoryFactory - 2 directories currently being tracked
[junit4:junit4] 2> 1791599 T1997 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1368907667871/jetty3 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1368907667871/jetty3;done=false>>]
[junit4:junit4] 2> 1791599 T1997 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1368907667871/jetty3
[junit4:junit4] 2> 1791599 T1997 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1368907667871/jetty3/index.20130518231610312 [CachedDir<<refCount=0;path=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1368907667871/jetty3/index.20130518231610312;done=false>>]
[junit4:junit4] 2> 1791600 T1997 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.SyncSliceTest-1368907667871/jetty3/index.20130518231610312
[junit4:junit4] 2> 1791600 T1997 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1791600 T1997 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1791601 T1997 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> 1791601 T1997 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
[junit4:junit4] 2> 1791601 T1997 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4:junit4] 2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=sr_CS, timezone=EAT
[junit4:junit4] 2> NOTE: FreeBSD 9.0-RELEASE amd64/Oracle Corporation 1.7.0_17 (64-bit)/cpus=16,threads=3,free=254355304,total=495976448
[junit4:junit4] 2> NOTE: All tests run in this JVM: [TestDistributedSearch, IndexReaderFactoryTest, TestLazyCores, PrimitiveFieldTypeTest, MultiTermTest, PreAnalyzedUpdateProcessorTest, QueryParsingTest, SimpleFacetsTest, TestHashPartitioner, TestDynamicFieldCollectionResource, DebugComponentTest, HighlighterConfigTest, TestReload, TestSchemaSimilarityResource, TestFoldingMultitermQuery, FieldAnalysisRequestHandlerTest, TestFieldTypeResource, RAMDirectoryFactoryTest, TestMultiCoreConfBootstrap, ShardRoutingCustomTest, LeaderElectionIntegrationTest, TestUtils, PluginInfoTest, AliasIntegrationTest, TestSolrCoreProperties, TestFaceting, TestBM25SimilarityFactory, PingRequestHandlerTest, DistanceFunctionTest, SpatialFilterTest, BadCopyFieldTest, TestZkChroot, ExternalFileFieldSortTest, UpdateParamsTest, CollectionsAPIDistributedZkTest, TestCSVLoader, TestMergePolicyConfig, TestSolrDeletionPolicy2, SolrRequestParserTest, TestLuceneMatchVersion, TestCSVResponseWriter, ShowFileRequestHandlerTest, TestRangeQuery, TestSolr4Spatial, TestSolrXml, TestPseudoReturnFields, UUIDFieldTest, DocumentAnalysisRequestHandlerTest, JsonLoaderTest, TestComponentsName, TestGroupingSearch, OutputWriterTest, SearchHandlerTest, NotRequiredUniqueKeyTest, HardAutoCommitTest, TestDocumentBuilder, TestLRUCache, TestSolrIndexConfig, ChaosMonkeyShardSplitTest, TestBadConfig, ZkSolrClientTest, RecoveryZkTest, FieldMutatingUpdateProcessorTest, TestElisionMultitermQuery, ShardRoutingTest, TestFieldCollectionResource, TestNumberUtils, NoCacheHeaderTest, TestDocSet, SignatureUpdateProcessorFactoryTest, XsltUpdateRequestHandlerTest, SyncSliceTest]
[junit4:junit4] Completed on J0 in 910.90s, 1 test, 1 failure <<< FAILURES!
[...truncated 430 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/build.xml:383: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/build.xml:363: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/build.xml:181: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/solr/common-build.xml:437: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/lucene/common-build.xml:1243: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-java7/lucene/common-build.xml:887: There were test failures: 296 suites, 1234 tests, 1 error, 1 failure, 27 ignored (8 assumptions)
Total time: 54 minutes 14 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure