You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@cassandra.apache.org by Hefeng Yuan <hf...@rhapsody.com> on 2011/08/09 23:24:26 UTC
AnitEntropy/MerkleTree Error
Hi,
We are seeing some AntiEntropy error in our production servers, since it's hard to repro in other env, pasting the stack track here for some clue.
We're using a cluster of 2 data centers, 9 nodes, 6 for online traffic, 3 for Brisk BI. Our RF is Cassandra:5, Brisk:1.
The exception stack is appended in the end.
Any help is appreciated.
Thanks,
Hefeng
ERROR [AntiEntropyStage:2] 2011-08-08 04:24:39,556 AbstractCassandraDaemon.java (line 113) Fatal exception in thread Thread[AntiEntropyStage:2,5,main]
java.lang.AssertionError
at org.apache.cassandra.utils.MerkleTree.inc(MerkleTree.java:154)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:262)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.difference(MerkleTree.java:244)
at org.apache.cassandra.service.AntiEntropyService$Differencer.run(AntiEntropyService.java:458)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
ERROR [AntiEntropyStage:1] 2011-08-08 04:24:39,580 AbstractCassandraDaemon.java (line 113) Fatal exception in thread Thread[AntiEntropyStage:1,5,main]
java.lang.AssertionError
at org.apache.cassandra.utils.MerkleTree.inc(MerkleTree.java:154)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:262)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
at org.apache.cassandra.utils.MerkleTree.difference(MerkleTree.java:244)
at org.apache.cassandra.service.AntiEntropyService$Differencer.run(AntiEntropyService.java:458)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:619)
Re: AnitEntropy/MerkleTree Error
Posted by Hefeng Yuan <hf...@rhapsody.com>.
Thanks for your clue, Sylvain, I've created ticket here: https://issues.apache.org/jira/browse/CASSANDRA-3014
Let me know if I need to provide more information.
On Aug 10, 2011, at 3:32 AM, Sylvain Lebresne wrote:
> Yeah, that is not supposed to happen.
> This could be related to
> https://issues.apache.org/jira/browse/CASSANDRA-2758
> if you are using something < 0.8.1 but I'm not sure.
>
> In any case, do you mind opening a JIRA ticket and to give the following
> informations:
> - Which version of Cassandra you are running?
> - Which partitionner you are using: RandomPartitionner or an order
> preserving one?
>
> --
> Sylvain
>
> On Tue, Aug 9, 2011 at 11:24 PM, Hefeng Yuan <hf...@rhapsody.com> wrote:
>> Hi,
>> We are seeing some AntiEntropy error in our production servers, since it's
>> hard to repro in other env, pasting the stack track here for some clue.
>> We're using a cluster of 2 data centers, 9 nodes, 6 for online traffic, 3
>> for Brisk BI. Our RF is Cassandra:5, Brisk:1.
>> The exception stack is appended in the end.
>> Any help is appreciated.
>> Thanks,
>> Hefeng
>>
>> ERROR [AntiEntropyStage:2] 2011-08-08 04:24:39,556
>> AbstractCassandraDaemon.java (line 113) Fatal exception in thread
>> Thread[AntiEntropyStage:2,5,main]
>> java.lang.AssertionError
>> at org.apache.cassandra.utils.MerkleTree.inc(MerkleTree.java:154)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:262)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at org.apache.cassandra.utils.MerkleTree.difference(MerkleTree.java:244)
>> at
>> org.apache.cassandra.service.AntiEntropyService$Differencer.run(AntiEntropyService.java:458)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
>> at java.lang.Thread.run(Thread.java:619)
>> ERROR [AntiEntropyStage:1] 2011-08-08 04:24:39,580
>> AbstractCassandraDaemon.java (line 113) Fatal exception in thread
>> Thread[AntiEntropyStage:1,5,main]
>> java.lang.AssertionError
>> at org.apache.cassandra.utils.MerkleTree.inc(MerkleTree.java:154)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:262)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at
>> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
>> at org.apache.cassandra.utils.MerkleTree.difference(MerkleTree.java:244)
>> at
>> org.apache.cassandra.service.AntiEntropyService$Differencer.run(AntiEntropyService.java:458)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
>> at java.lang.Thread.run(Thread.java:619)
Re: AnitEntropy/MerkleTree Error
Posted by Sylvain Lebresne <sy...@datastax.com>.
Yeah, that is not supposed to happen.
This could be related to
https://issues.apache.org/jira/browse/CASSANDRA-2758
if you are using something < 0.8.1 but I'm not sure.
In any case, do you mind opening a JIRA ticket and to give the following
informations:
- Which version of Cassandra you are running?
- Which partitionner you are using: RandomPartitionner or an order
preserving one?
--
Sylvain
On Tue, Aug 9, 2011 at 11:24 PM, Hefeng Yuan <hf...@rhapsody.com> wrote:
> Hi,
> We are seeing some AntiEntropy error in our production servers, since it's
> hard to repro in other env, pasting the stack track here for some clue.
> We're using a cluster of 2 data centers, 9 nodes, 6 for online traffic, 3
> for Brisk BI. Our RF is Cassandra:5, Brisk:1.
> The exception stack is appended in the end.
> Any help is appreciated.
> Thanks,
> Hefeng
>
> ERROR [AntiEntropyStage:2] 2011-08-08 04:24:39,556
> AbstractCassandraDaemon.java (line 113) Fatal exception in thread
> Thread[AntiEntropyStage:2,5,main]
> java.lang.AssertionError
> at org.apache.cassandra.utils.MerkleTree.inc(MerkleTree.java:154)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:262)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at org.apache.cassandra.utils.MerkleTree.difference(MerkleTree.java:244)
> at
> org.apache.cassandra.service.AntiEntropyService$Differencer.run(AntiEntropyService.java:458)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:619)
> ERROR [AntiEntropyStage:1] 2011-08-08 04:24:39,580
> AbstractCassandraDaemon.java (line 113) Fatal exception in thread
> Thread[AntiEntropyStage:1,5,main]
> java.lang.AssertionError
> at org.apache.cassandra.utils.MerkleTree.inc(MerkleTree.java:154)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:262)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:284)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at
> org.apache.cassandra.utils.MerkleTree.differenceHelper(MerkleTree.java:273)
> at org.apache.cassandra.utils.MerkleTree.difference(MerkleTree.java:244)
> at
> org.apache.cassandra.service.AntiEntropyService$Differencer.run(AntiEntropyService.java:458)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
> at java.lang.Thread.run(Thread.java:619)