You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@cassandra.apache.org by Chris Were <ch...@gmail.com> on 2009/10/06 00:32:17 UTC
Cassandra Fatal Exceptions
Hi,
I updated from trunk two days ago, but ever since I've been receiving a
series of Fatal Exceptions. My logs show that about an hour after starting
Cassandra I receive a RejectedExecutionException and two Assertion
exceptions. From that point on, the log files just fill up with repeated
TimeoutExceptions (each one followed by two Assertion exceptions). I am in
no way familar with Cassandra internals so am at a loss to determine what's
happening here.
Here is an excerpt of the first 6 exceptions (the last three continue ad
infinitum):
ERROR [FLUSH-TIMER] 2009-10-05 05:04:14,465 CassandraDaemon.java (line
org.apache.cassandra.service.CassandraDaemon) Fatal exception in thread
Thread[FLUSH-TIMER,5,main]
java.util.concurrent.RejectedExecutionException
at
java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
at
java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
at
java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
at
java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:106)
at
org.apache.cassandra.db.ColumnFamilyStore.submitFlush(ColumnFamilyStore.java:926)
at
org.apache.cassandra.db.ColumnFamilyStore.switchMemtable(ColumnFamilyStore.java:367)
at
org.apache.cassandra.db.ColumnFamilyStore.forceFlush(ColumnFamilyStore.java:415)
at
org.apache.cassandra.db.ColumnFamilyStore.forceFlushIfExpired(ColumnFamilyStore.java:407)
at org.apache.cassandra.db.Table$1.run(Table.java:529)
at java.util.TimerThread.mainLoop(Timer.java:534)
at java.util.TimerThread.run(Timer.java:484)
ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574
DebuggableThreadPoolExecutor.java (line
org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
ThreadPoolExecutor
java.lang.AssertionError
at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
at
org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
at org.apache.cassandra.db.Table.apply(Table.java:626)
at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
at
org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574 CassandraDaemon.java
(line org.apache.cassandra.service.CassandraDaemon) Fatal exception
in thread Thread[ROW-MUTATION-STAGE:6,5,main]
java.lang.AssertionError
at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
at
org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
at org.apache.cassandra.db.Table.apply(Table.java:626)
at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
at
org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
ERROR [pool-1-thread-1] 2009-10-05 05:04:19,525 StorageProxy.java (line
org.apache.cassandra.service.StorageProxy) error writing key
ec9787399d444098819180163b16abc4
java.util.concurrent.TimeoutException: Operation timed out - received only 0
responses from .
at
org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:92)
at
org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:165)
at
org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
at
org.apache.cassandra.service.CassandraServer.batch_insert(CassandraServer.java:448)
at
org.apache.cassandra.service.Cassandra$Processor$batch_insert.process(Cassandra.java:854)
at
org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
at
org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784
DebuggableThreadPoolExecutor.java (line
org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
ThreadPoolExecutor
java.lang.AssertionError
at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
at
org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
at org.apache.cassandra.db.Table.apply(Table.java:626)
at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
at
org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784 CassandraDaemon.java
(line org.apache.cassandra.service.CassandraDaemon) Fatal exception in
thread Thread[ROW-MUTATION-STAGE:2,5,main]
java.lang.AssertionError
at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
at
org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
at org.apache.cassandra.db.Table.apply(Table.java:626)
at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
at
org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
at
org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:636)
Cheers,
Chris
Re: Cassandra Fatal Exceptions
Posted by Chris Were <ch...@gmail.com>.
Have had this running for almost 8 hours without fault, so it looks good.
Cheers,
Chris
On Wed, Oct 7, 2009 at 3:17 AM, Jonathan Ellis <jb...@gmail.com> wrote:
> Patch attached to #471, if you'd like to test.
>
> -Jonathan
>
> On Mon, Oct 5, 2009 at 6:28 PM, Chris Were <ch...@gmail.com> wrote:
> > Thanks -- I'll head back to 0.4.0 then :)
> > Cheers.
> >
> > On Tue, Oct 6, 2009 at 9:11 AM, Jonathan Ellis <jb...@gmail.com>
> wrote:
> >>
> >> Dan beat you to it: http://issues.apache.org/jira/browse/CASSANDRA-471
> >>
> >> On Mon, Oct 5, 2009 at 5:39 PM, Jonathan Ellis <jb...@gmail.com>
> wrote:
> >> > looks like you ran into a regression. that's the risk you run on
> trunk.
> >> > :)
> >> >
> >> > Could you open a ticket for this on
> >> > https://issues.apache.org/jira/browse/CASSANDRA ?
> >> >
> >> > -Jonathan
> >> >
> >> > On Mon, Oct 5, 2009 at 5:32 PM, Chris Were <ch...@gmail.com>
> wrote:
> >> >> Hi,
> >> >> I updated from trunk two days ago, but ever since I've been receiving
> a
> >> >> series of Fatal Exceptions. My logs show that about an hour after
> >> >> starting
> >> >> Cassandra I receive a RejectedExecutionException and two Assertion
> >> >> exceptions. From that point on, the log files just fill up with
> >> >> repeated
> >> >> TimeoutExceptions (each one followed by two Assertion exceptions). I
> am
> >> >> in
> >> >> no way familar with Cassandra internals so am at a loss to determine
> >> >> what's
> >> >> happening here.
> >> >> Here is an excerpt of the first 6 exceptions (the last three continue
> >> >> ad
> >> >> infinitum):
> >> >> ERROR [FLUSH-TIMER] 2009-10-05 05:04:14,465 CassandraDaemon.java
> (line
> >> >> org.apache.cassandra.service.CassandraDaemon) Fatal exception in
> thread
> >> >> Thread[FLUSH-TIMER,5,main]
> >> >> java.util.concurrent.RejectedExecutionException
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:106)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.ColumnFamilyStore.submitFlush(ColumnFamilyStore.java:926)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.ColumnFamilyStore.switchMemtable(ColumnFamilyStore.java:367)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.ColumnFamilyStore.forceFlush(ColumnFamilyStore.java:415)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.ColumnFamilyStore.forceFlushIfExpired(ColumnFamilyStore.java:407)
> >> >> at org.apache.cassandra.db.Table$1.run(Table.java:529)
> >> >> at java.util.TimerThread.mainLoop(Timer.java:534)
> >> >> at java.util.TimerThread.run(Timer.java:484)
> >> >> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574
> >> >> DebuggableThreadPoolExecutor.java (line
> >> >> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error
> in
> >> >> ThreadPoolExecutor
> >> >> java.lang.AssertionError
> >> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> >> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
> >> >> at
> >> >> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> >> at java.lang.Thread.run(Thread.java:636)
> >> >> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574
> >> >> CassandraDaemon.java
> >> >> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception
> >> >> in thread Thread[ROW-MUTATION-STAGE:6,5,main]
> >> >> java.lang.AssertionError
> >> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> >> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
> >> >> at
> >> >> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> >> at java.lang.Thread.run(Thread.java:636)
> >> >> ERROR [pool-1-thread-1] 2009-10-05 05:04:19,525 StorageProxy.java
> (line
> >> >> org.apache.cassandra.service.StorageProxy) error writing key
> >> >> ec9787399d444098819180163b16abc4
> >> >> java.util.concurrent.TimeoutException: Operation timed out - received
> >> >> only 0
> >> >> responses from .
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:92)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:165)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.service.CassandraServer.batch_insert(CassandraServer.java:448)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.service.Cassandra$Processor$batch_insert.process(Cassandra.java:854)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
> >> >> at
> >> >>
> >> >>
> org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> >> at java.lang.Thread.run(Thread.java:636)
> >> >> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784
> >> >> DebuggableThreadPoolExecutor.java (line
> >> >> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error
> in
> >> >> ThreadPoolExecutor
> >> >> java.lang.AssertionError
> >> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> >> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
> >> >> at
> >> >> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> >> at java.lang.Thread.run(Thread.java:636)
> >> >> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784
> >> >> CassandraDaemon.java
> >> >> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception
> in
> >> >> thread Thread[ROW-MUTATION-STAGE:2,5,main]
> >> >> java.lang.AssertionError
> >> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> >> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
> >> >> at
> >> >> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> >> >> at
> >> >>
> >> >>
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> >> at
> >> >>
> >> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> >> at java.lang.Thread.run(Thread.java:636)
> >> >> Cheers,
> >> >> Chris
> >> >
> >
> >
>
Re: Cassandra Fatal Exceptions
Posted by Jonathan Ellis <jb...@gmail.com>.
Patch attached to #471, if you'd like to test.
-Jonathan
On Mon, Oct 5, 2009 at 6:28 PM, Chris Were <ch...@gmail.com> wrote:
> Thanks -- I'll head back to 0.4.0 then :)
> Cheers.
>
> On Tue, Oct 6, 2009 at 9:11 AM, Jonathan Ellis <jb...@gmail.com> wrote:
>>
>> Dan beat you to it: http://issues.apache.org/jira/browse/CASSANDRA-471
>>
>> On Mon, Oct 5, 2009 at 5:39 PM, Jonathan Ellis <jb...@gmail.com> wrote:
>> > looks like you ran into a regression. that's the risk you run on trunk.
>> > :)
>> >
>> > Could you open a ticket for this on
>> > https://issues.apache.org/jira/browse/CASSANDRA ?
>> >
>> > -Jonathan
>> >
>> > On Mon, Oct 5, 2009 at 5:32 PM, Chris Were <ch...@gmail.com> wrote:
>> >> Hi,
>> >> I updated from trunk two days ago, but ever since I've been receiving a
>> >> series of Fatal Exceptions. My logs show that about an hour after
>> >> starting
>> >> Cassandra I receive a RejectedExecutionException and two Assertion
>> >> exceptions. From that point on, the log files just fill up with
>> >> repeated
>> >> TimeoutExceptions (each one followed by two Assertion exceptions). I am
>> >> in
>> >> no way familar with Cassandra internals so am at a loss to determine
>> >> what's
>> >> happening here.
>> >> Here is an excerpt of the first 6 exceptions (the last three continue
>> >> ad
>> >> infinitum):
>> >> ERROR [FLUSH-TIMER] 2009-10-05 05:04:14,465 CassandraDaemon.java (line
>> >> org.apache.cassandra.service.CassandraDaemon) Fatal exception in thread
>> >> Thread[FLUSH-TIMER,5,main]
>> >> java.util.concurrent.RejectedExecutionException
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
>> >> at
>> >>
>> >> java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:106)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.submitFlush(ColumnFamilyStore.java:926)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.switchMemtable(ColumnFamilyStore.java:367)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.forceFlush(ColumnFamilyStore.java:415)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.forceFlushIfExpired(ColumnFamilyStore.java:407)
>> >> at org.apache.cassandra.db.Table$1.run(Table.java:529)
>> >> at java.util.TimerThread.mainLoop(Timer.java:534)
>> >> at java.util.TimerThread.run(Timer.java:484)
>> >> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574
>> >> DebuggableThreadPoolExecutor.java (line
>> >> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
>> >> ThreadPoolExecutor
>> >> java.lang.AssertionError
>> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
>> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
>> >> at
>> >> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
>> >> at
>> >>
>> >> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
>> >> at
>> >>
>> >> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> >> at java.lang.Thread.run(Thread.java:636)
>> >> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574
>> >> CassandraDaemon.java
>> >> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception
>> >> in thread Thread[ROW-MUTATION-STAGE:6,5,main]
>> >> java.lang.AssertionError
>> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
>> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
>> >> at
>> >> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
>> >> at
>> >>
>> >> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
>> >> at
>> >>
>> >> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> >> at java.lang.Thread.run(Thread.java:636)
>> >> ERROR [pool-1-thread-1] 2009-10-05 05:04:19,525 StorageProxy.java (line
>> >> org.apache.cassandra.service.StorageProxy) error writing key
>> >> ec9787399d444098819180163b16abc4
>> >> java.util.concurrent.TimeoutException: Operation timed out - received
>> >> only 0
>> >> responses from .
>> >> at
>> >>
>> >> org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:92)
>> >> at
>> >>
>> >> org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:165)
>> >> at
>> >>
>> >> org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
>> >> at
>> >>
>> >> org.apache.cassandra.service.CassandraServer.batch_insert(CassandraServer.java:448)
>> >> at
>> >>
>> >> org.apache.cassandra.service.Cassandra$Processor$batch_insert.process(Cassandra.java:854)
>> >> at
>> >>
>> >> org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
>> >> at
>> >>
>> >> org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> >> at java.lang.Thread.run(Thread.java:636)
>> >> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784
>> >> DebuggableThreadPoolExecutor.java (line
>> >> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
>> >> ThreadPoolExecutor
>> >> java.lang.AssertionError
>> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
>> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
>> >> at
>> >> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
>> >> at
>> >>
>> >> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
>> >> at
>> >>
>> >> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> >> at java.lang.Thread.run(Thread.java:636)
>> >> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784
>> >> CassandraDaemon.java
>> >> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception in
>> >> thread Thread[ROW-MUTATION-STAGE:2,5,main]
>> >> java.lang.AssertionError
>> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
>> >> at
>> >>
>> >> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
>> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
>> >> at
>> >> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
>> >> at
>> >>
>> >> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
>> >> at
>> >>
>> >> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> >> at
>> >>
>> >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> >> at java.lang.Thread.run(Thread.java:636)
>> >> Cheers,
>> >> Chris
>> >
>
>
Re: Cassandra Fatal Exceptions
Posted by Chris Were <ch...@gmail.com>.
Thanks -- I'll head back to 0.4.0 then :)
Cheers.
On Tue, Oct 6, 2009 at 9:11 AM, Jonathan Ellis <jb...@gmail.com> wrote:
> Dan beat you to it: http://issues.apache.org/jira/browse/CASSANDRA-471
>
> On Mon, Oct 5, 2009 at 5:39 PM, Jonathan Ellis <jb...@gmail.com> wrote:
> > looks like you ran into a regression. that's the risk you run on trunk.
> :)
> >
> > Could you open a ticket for this on
> > https://issues.apache.org/jira/browse/CASSANDRA ?
> >
> > -Jonathan
> >
> > On Mon, Oct 5, 2009 at 5:32 PM, Chris Were <ch...@gmail.com> wrote:
> >> Hi,
> >> I updated from trunk two days ago, but ever since I've been receiving a
> >> series of Fatal Exceptions. My logs show that about an hour after
> starting
> >> Cassandra I receive a RejectedExecutionException and two Assertion
> >> exceptions. From that point on, the log files just fill up with repeated
> >> TimeoutExceptions (each one followed by two Assertion exceptions). I am
> in
> >> no way familar with Cassandra internals so am at a loss to determine
> what's
> >> happening here.
> >> Here is an excerpt of the first 6 exceptions (the last three continue ad
> >> infinitum):
> >> ERROR [FLUSH-TIMER] 2009-10-05 05:04:14,465 CassandraDaemon.java (line
> >> org.apache.cassandra.service.CassandraDaemon) Fatal exception in thread
> >> Thread[FLUSH-TIMER,5,main]
> >> java.util.concurrent.RejectedExecutionException
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
> >> at
> >>
> java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:106)
> >> at
> >>
> org.apache.cassandra.db.ColumnFamilyStore.submitFlush(ColumnFamilyStore.java:926)
> >> at
> >>
> org.apache.cassandra.db.ColumnFamilyStore.switchMemtable(ColumnFamilyStore.java:367)
> >> at
> >>
> org.apache.cassandra.db.ColumnFamilyStore.forceFlush(ColumnFamilyStore.java:415)
> >> at
> >>
> org.apache.cassandra.db.ColumnFamilyStore.forceFlushIfExpired(ColumnFamilyStore.java:407)
> >> at org.apache.cassandra.db.Table$1.run(Table.java:529)
> >> at java.util.TimerThread.mainLoop(Timer.java:534)
> >> at java.util.TimerThread.run(Timer.java:484)
> >> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574
> >> DebuggableThreadPoolExecutor.java (line
> >> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
> >> ThreadPoolExecutor
> >> java.lang.AssertionError
> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> >> at
> >>
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
> >> at
> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> >> at
> >>
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> >> at
> >>
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> at java.lang.Thread.run(Thread.java:636)
> >> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574
> CassandraDaemon.java
> >> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception
> >> in thread Thread[ROW-MUTATION-STAGE:6,5,main]
> >> java.lang.AssertionError
> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> >> at
> >>
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
> >> at
> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> >> at
> >>
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> >> at
> >>
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> at java.lang.Thread.run(Thread.java:636)
> >> ERROR [pool-1-thread-1] 2009-10-05 05:04:19,525 StorageProxy.java (line
> >> org.apache.cassandra.service.StorageProxy) error writing key
> >> ec9787399d444098819180163b16abc4
> >> java.util.concurrent.TimeoutException: Operation timed out - received
> only 0
> >> responses from .
> >> at
> >>
> org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:92)
> >> at
> >>
> org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:165)
> >> at
> >>
> org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
> >> at
> >>
> org.apache.cassandra.service.CassandraServer.batch_insert(CassandraServer.java:448)
> >> at
> >>
> org.apache.cassandra.service.Cassandra$Processor$batch_insert.process(Cassandra.java:854)
> >> at
> >>
> org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
> >> at
> >>
> org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> at java.lang.Thread.run(Thread.java:636)
> >> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784
> >> DebuggableThreadPoolExecutor.java (line
> >> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
> >> ThreadPoolExecutor
> >> java.lang.AssertionError
> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> >> at
> >>
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
> >> at
> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> >> at
> >>
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> >> at
> >>
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> at java.lang.Thread.run(Thread.java:636)
> >> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784
> CassandraDaemon.java
> >> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception in
> >> thread Thread[ROW-MUTATION-STAGE:2,5,main]
> >> java.lang.AssertionError
> >> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> >> at
> >>
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> >> at org.apache.cassandra.db.Table.apply(Table.java:626)
> >> at
> org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> >> at
> >>
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> >> at
> >>
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> >> at
> >>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> >> at java.lang.Thread.run(Thread.java:636)
> >> Cheers,
> >> Chris
> >
>
Re: Cassandra Fatal Exceptions
Posted by Jonathan Ellis <jb...@gmail.com>.
Dan beat you to it: http://issues.apache.org/jira/browse/CASSANDRA-471
On Mon, Oct 5, 2009 at 5:39 PM, Jonathan Ellis <jb...@gmail.com> wrote:
> looks like you ran into a regression. that's the risk you run on trunk. :)
>
> Could you open a ticket for this on
> https://issues.apache.org/jira/browse/CASSANDRA ?
>
> -Jonathan
>
> On Mon, Oct 5, 2009 at 5:32 PM, Chris Were <ch...@gmail.com> wrote:
>> Hi,
>> I updated from trunk two days ago, but ever since I've been receiving a
>> series of Fatal Exceptions. My logs show that about an hour after starting
>> Cassandra I receive a RejectedExecutionException and two Assertion
>> exceptions. From that point on, the log files just fill up with repeated
>> TimeoutExceptions (each one followed by two Assertion exceptions). I am in
>> no way familar with Cassandra internals so am at a loss to determine what's
>> happening here.
>> Here is an excerpt of the first 6 exceptions (the last three continue ad
>> infinitum):
>> ERROR [FLUSH-TIMER] 2009-10-05 05:04:14,465 CassandraDaemon.java (line
>> org.apache.cassandra.service.CassandraDaemon) Fatal exception in thread
>> Thread[FLUSH-TIMER,5,main]
>> java.util.concurrent.RejectedExecutionException
>> at
>> java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
>> at
>> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
>> at
>> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
>> at
>> java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:106)
>> at
>> org.apache.cassandra.db.ColumnFamilyStore.submitFlush(ColumnFamilyStore.java:926)
>> at
>> org.apache.cassandra.db.ColumnFamilyStore.switchMemtable(ColumnFamilyStore.java:367)
>> at
>> org.apache.cassandra.db.ColumnFamilyStore.forceFlush(ColumnFamilyStore.java:415)
>> at
>> org.apache.cassandra.db.ColumnFamilyStore.forceFlushIfExpired(ColumnFamilyStore.java:407)
>> at org.apache.cassandra.db.Table$1.run(Table.java:529)
>> at java.util.TimerThread.mainLoop(Timer.java:534)
>> at java.util.TimerThread.run(Timer.java:484)
>> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574
>> DebuggableThreadPoolExecutor.java (line
>> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
>> ThreadPoolExecutor
>> java.lang.AssertionError
>> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
>> at
>> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
>> at org.apache.cassandra.db.Table.apply(Table.java:626)
>> at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
>> at
>> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
>> at
>> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> at java.lang.Thread.run(Thread.java:636)
>> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574 CassandraDaemon.java
>> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception
>> in thread Thread[ROW-MUTATION-STAGE:6,5,main]
>> java.lang.AssertionError
>> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
>> at
>> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
>> at org.apache.cassandra.db.Table.apply(Table.java:626)
>> at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
>> at
>> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
>> at
>> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> at java.lang.Thread.run(Thread.java:636)
>> ERROR [pool-1-thread-1] 2009-10-05 05:04:19,525 StorageProxy.java (line
>> org.apache.cassandra.service.StorageProxy) error writing key
>> ec9787399d444098819180163b16abc4
>> java.util.concurrent.TimeoutException: Operation timed out - received only 0
>> responses from .
>> at
>> org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:92)
>> at
>> org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:165)
>> at
>> org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
>> at
>> org.apache.cassandra.service.CassandraServer.batch_insert(CassandraServer.java:448)
>> at
>> org.apache.cassandra.service.Cassandra$Processor$batch_insert.process(Cassandra.java:854)
>> at
>> org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
>> at
>> org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> at java.lang.Thread.run(Thread.java:636)
>> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784
>> DebuggableThreadPoolExecutor.java (line
>> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
>> ThreadPoolExecutor
>> java.lang.AssertionError
>> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
>> at
>> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
>> at org.apache.cassandra.db.Table.apply(Table.java:626)
>> at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
>> at
>> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
>> at
>> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> at java.lang.Thread.run(Thread.java:636)
>> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784 CassandraDaemon.java
>> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception in
>> thread Thread[ROW-MUTATION-STAGE:2,5,main]
>> java.lang.AssertionError
>> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
>> at
>> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
>> at org.apache.cassandra.db.Table.apply(Table.java:626)
>> at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
>> at
>> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
>> at
>> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
>> at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
>> at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
>> at java.lang.Thread.run(Thread.java:636)
>> Cheers,
>> Chris
>
Re: Cassandra Fatal Exceptions
Posted by Jonathan Ellis <jb...@gmail.com>.
looks like you ran into a regression. that's the risk you run on trunk. :)
Could you open a ticket for this on
https://issues.apache.org/jira/browse/CASSANDRA ?
-Jonathan
On Mon, Oct 5, 2009 at 5:32 PM, Chris Were <ch...@gmail.com> wrote:
> Hi,
> I updated from trunk two days ago, but ever since I've been receiving a
> series of Fatal Exceptions. My logs show that about an hour after starting
> Cassandra I receive a RejectedExecutionException and two Assertion
> exceptions. From that point on, the log files just fill up with repeated
> TimeoutExceptions (each one followed by two Assertion exceptions). I am in
> no way familar with Cassandra internals so am at a loss to determine what's
> happening here.
> Here is an excerpt of the first 6 exceptions (the last three continue ad
> infinitum):
> ERROR [FLUSH-TIMER] 2009-10-05 05:04:14,465 CassandraDaemon.java (line
> org.apache.cassandra.service.CassandraDaemon) Fatal exception in thread
> Thread[FLUSH-TIMER,5,main]
> java.util.concurrent.RejectedExecutionException
> at
> java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:1956)
> at
> java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:816)
> at
> java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1337)
> at
> java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:106)
> at
> org.apache.cassandra.db.ColumnFamilyStore.submitFlush(ColumnFamilyStore.java:926)
> at
> org.apache.cassandra.db.ColumnFamilyStore.switchMemtable(ColumnFamilyStore.java:367)
> at
> org.apache.cassandra.db.ColumnFamilyStore.forceFlush(ColumnFamilyStore.java:415)
> at
> org.apache.cassandra.db.ColumnFamilyStore.forceFlushIfExpired(ColumnFamilyStore.java:407)
> at org.apache.cassandra.db.Table$1.run(Table.java:529)
> at java.util.TimerThread.mainLoop(Timer.java:534)
> at java.util.TimerThread.run(Timer.java:484)
> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574
> DebuggableThreadPoolExecutor.java (line
> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
> ThreadPoolExecutor
> java.lang.AssertionError
> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> at
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> at org.apache.cassandra.db.Table.apply(Table.java:626)
> at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> at
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:636)
> ERROR [ROW-MUTATION-STAGE:6] 2009-10-05 05:04:14,574 CassandraDaemon.java
> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception
> in thread Thread[ROW-MUTATION-STAGE:6,5,main]
> java.lang.AssertionError
> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> at
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> at org.apache.cassandra.db.Table.apply(Table.java:626)
> at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> at
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:636)
> ERROR [pool-1-thread-1] 2009-10-05 05:04:19,525 StorageProxy.java (line
> org.apache.cassandra.service.StorageProxy) error writing key
> ec9787399d444098819180163b16abc4
> java.util.concurrent.TimeoutException: Operation timed out - received only 0
> responses from .
> at
> org.apache.cassandra.service.QuorumResponseHandler.get(QuorumResponseHandler.java:92)
> at
> org.apache.cassandra.service.StorageProxy.insertBlocking(StorageProxy.java:165)
> at
> org.apache.cassandra.service.CassandraServer.doInsert(CassandraServer.java:468)
> at
> org.apache.cassandra.service.CassandraServer.batch_insert(CassandraServer.java:448)
> at
> org.apache.cassandra.service.Cassandra$Processor$batch_insert.process(Cassandra.java:854)
> at
> org.apache.cassandra.service.Cassandra$Processor.process(Cassandra.java:627)
> at
> org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:253)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:636)
> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784
> DebuggableThreadPoolExecutor.java (line
> org.apache.cassandra.concurrent.DebuggableThreadPoolExecutor) Error in
> ThreadPoolExecutor
> java.lang.AssertionError
> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> at
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> at org.apache.cassandra.db.Table.apply(Table.java:626)
> at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> at
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:636)
> ERROR [ROW-MUTATION-STAGE:2] 2009-10-05 05:04:19,784 CassandraDaemon.java
> (line org.apache.cassandra.service.CassandraDaemon) Fatal exception in
> thread Thread[ROW-MUTATION-STAGE:2,5,main]
> java.lang.AssertionError
> at org.apache.cassandra.db.Memtable.put(Memtable.java:143)
> at
> org.apache.cassandra.db.ColumnFamilyStore.apply(ColumnFamilyStore.java:442)
> at org.apache.cassandra.db.Table.apply(Table.java:626)
> at org.apache.cassandra.db.RowMutation.apply(RowMutation.java:205)
> at
> org.apache.cassandra.db.RowMutationVerbHandler.doVerb(RowMutationVerbHandler.java:61)
> at
> org.apache.cassandra.net.MessageDeliveryTask.run(MessageDeliveryTask.java:39)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
> at java.lang.Thread.run(Thread.java:636)
> Cheers,
> Chris