You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@geode.apache.org by "Owen Nichols (Jira)" <ji...@apache.org> on 2022/06/22 20:47:03 UTC
[jira] [Closed] (GEODE-9204) A not serializable exception can cause a ServerConnection thread to get stuck waiting for a reply from another member
[ https://issues.apache.org/jira/browse/GEODE-9204?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Owen Nichols closed GEODE-9204.
-------------------------------
> A not serializable exception can cause a ServerConnection thread to get stuck waiting for a reply from another member
> ---------------------------------------------------------------------------------------------------------------------
>
> Key: GEODE-9204
> URL: https://issues.apache.org/jira/browse/GEODE-9204
> Project: Geode
> Issue Type: Bug
> Components: membership, messaging
> Reporter: Bruce J Schuchardt
> Assignee: Kamilla Aslami
> Priority: Major
> Labels: pull-request-available
> Fix For: 1.15.0
>
>
> A test case that reproduces it is:
> - a client get request is received in one server and sent to another server
> - the other server uses a CacheLoader to load the value
> - the CacheLoader throws an exception containing a non-serializable object
> - the reply attempts to serialize that exception but fails with NotSerializableException
> - the original server's ServerConnection thread gets stuck waiting for a reply that will never come
> Here is a stack trace showing the NotSerializableException:
> {noformat}
> [severe 2018/03/20 14:30:27.793 PDT <P2P message reader for elgreco(85544)<v11>:30177 unshared ordered uid=14 dom #1 port=53923> tid=0x5c] Uncaught exception processing partitioned.GetMessage(prid=2 (name = "/data") processorId=0; posDup=false; key=0; callback arg=null; context=identity(elgreco(client:85552:loner):53907:fce35145:client,connection=2)
> org.apache.geode.InternalGemFireException: java.io.NotSerializableException: java.lang.Object
> at org.apache.geode.internal.tcp.DirectReplySender.putOutgoing(DirectReplySender.java:76)
> at org.apache.geode.distributed.internal.ReplyMessage.send(ReplyMessage.java:109)
> at org.apache.geode.internal.cache.partitioned.PartitionMessage.sendReply(PartitionMessage.java:392)
> at org.apache.geode.internal.cache.partitioned.PartitionMessage.process(PartitionMessage.java:376)
> at org.apache.geode.distributed.internal.DistributionMessage.scheduleAction(DistributionMessage.java:386)
> at org.apache.geode.distributed.internal.DistributionMessage.schedule(DistributionMessage.java:449)
> at org.apache.geode.distributed.internal.DistributionManager.scheduleIncomingMessage(DistributionManager.java:3872)
> at org.apache.geode.distributed.internal.DistributionManager.handleIncomingDMsg(DistributionManager.java:3496)
> at org.apache.geode.distributed.internal.DistributionManager$MyListener.messageReceived(DistributionManager.java:4693)
> at org.apache.geode.distributed.internal.membership.jgroup.JGroupMembershipManager.processMessage(JGroupMembershipManager.java:2128)
> at org.apache.geode.distributed.internal.membership.jgroup.JGroupMembershipManager.handleOrDeferMessage(JGroupMembershipManager.java:2037)
> at org.apache.geode.distributed.internal.membership.jgroup.JGroupMembershipManager$MyDCReceiver.messageReceived(JGroupMembershipManager.java:647)
> at org.apache.geode.distributed.internal.direct.DirectChannel.receive(DirectChannel.java:804)
> at org.apache.geode.internal.tcp.TCPConduit.messageReceived(TCPConduit.java:835)
> at org.apache.geode.internal.tcp.Connection.dispatchMessage(Connection.java:3932)
> at org.apache.geode.internal.tcp.Connection.processNIOBuffer(Connection.java:3515)
> at org.apache.geode.internal.tcp.Connection.runNioReader(Connection.java:1827)
> at org.apache.geode.internal.tcp.Connection.run(Connection.java:1702)
> at java.lang.Thread.run(Thread.java:745)
> Caused by: java.io.NotSerializableException: java.lang.Object
> at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1184)
> at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
> at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1509)
> at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
> at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
> at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
> at java.io.ObjectOutputStream.defaultWriteObject(ObjectOutputStream.java:441)
> at java.lang.Throwable.writeObject(Throwable.java:985)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:1028)
> at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
> at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
> at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
> at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1548)
> at java.io.ObjectOutputStream.defaultWriteObject(ObjectOutputStream.java:441)
> at java.lang.Throwable.writeObject(Throwable.java:985)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at java.io.ObjectStreamClass.invokeWriteObject(ObjectStreamClass.java:1028)
> at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1496)
> at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1432)
> at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1178)
> at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:348)
> at org.apache.geode.internal.InternalDataSerializer.writeSerializableObject(InternalDataSerializer.java:2352)
> at org.apache.geode.internal.InternalDataSerializer.basicWriteObject(InternalDataSerializer.java:2226)
> at org.apache.geode.DataSerializer.writeObject(DataSerializer.java:3179)
> at org.apache.geode.distributed.internal.ReplyMessage.toData(ReplyMessage.java:305)
> at org.apache.geode.internal.InternalDataSerializer.invokeToData(InternalDataSerializer.java:2402)
> at org.apache.geode.internal.InternalDataSerializer.writeDSFID(InternalDataSerializer.java:1373)
> at org.apache.geode.internal.tcp.MsgStreamer.writeMessage(MsgStreamer.java:239)
> at org.apache.geode.internal.tcp.DirectReplySender.putOutgoing(DirectReplySender.java:64)
> ... 18 more
> {noformat}
> Here is where the ServerConnection thread is stuck:
> {noformat}
> "ServerConnection on port 53895 Thread 1" #89 prio=5 os_prio=31 tid=0x00007f85d9001800 nid=0xcd07 runnable [0x000070000fddd000]
> java.lang.Thread.State: RUNNABLE
> at sun.nio.ch.FileDispatcherImpl.read0(Native Method)
> at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39)
> at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223)
> at sun.nio.ch.IOUtil.read(IOUtil.java:192)
> at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380)
> - locked <0x0000000780c82c28> (a java.lang.Object)
> at org.apache.geode.internal.tcp.NIOMsgReader.readAtLeast(NIOMsgReader.java:45)
> at org.apache.geode.internal.tcp.MsgReader.readHeader(MsgReader.java:38)
> at org.apache.geode.internal.tcp.Connection.readAck(Connection.java:3365)
> at org.apache.geode.distributed.internal.direct.DirectChannel.readAcks(DirectChannel.java:567)
> at org.apache.geode.distributed.internal.direct.DirectChannel.sendToMany(DirectChannel.java:499)
> at org.apache.geode.distributed.internal.direct.DirectChannel.sendToOne(DirectChannel.java:310)
> at org.apache.geode.distributed.internal.direct.DirectChannel.send(DirectChannel.java:696)
> at org.apache.geode.distributed.internal.membership.jgroup.JGroupMembershipManager.directChannelSend(JGroupMembershipManager.java:2844)
> at org.apache.geode.distributed.internal.membership.jgroup.JGroupMembershipManager.send(JGroupMembershipManager.java:3078)
> at org.apache.geode.distributed.internal.DistributionChannel.send(DistributionChannel.java:79)
> at org.apache.geode.distributed.internal.DistributionManager.sendOutgoing(DistributionManager.java:3780)
> at org.apache.geode.distributed.internal.DistributionManager.sendMessage(DistributionManager.java:3821)
> at org.apache.geode.distributed.internal.DistributionManager.putOutgoing(DistributionManager.java:1957)
> at org.apache.geode.internal.cache.partitioned.GetMessage.send(GetMessage.java:258)
> at org.apache.geode.internal.cache.PartitionedRegion.getRemotely(PartitionedRegion.java:4990)
> at org.apache.geode.internal.cache.PartitionedRegion.getFromBucket(PartitionedRegion.java:4096)
> at org.apache.geode.internal.cache.PartitionedRegion.findObjectInSystem(PartitionedRegion.java:3459)
> at org.apache.geode.internal.cache.PartitionedRegionDataView.findObject(PartitionedRegionDataView.java:65)
> at org.apache.geode.internal.cache.PartitionedRegion.get(PartitionedRegion.java:3241)
> at org.apache.geode.internal.cache.tier.sockets.command.Get70.getValueAndIsObject(Get70.java:271)
> at org.apache.geode.internal.cache.tier.sockets.command.Get70.cmdExecute(Get70.java:162)
> at org.apache.geode.internal.cache.tier.sockets.BaseCommand.execute(BaseCommand.java:182)
> at org.apache.geode.internal.cache.tier.sockets.ServerConnection.doNormalMsg(ServerConnection.java:789)
> at org.apache.geode.internal.cache.tier.sockets.ServerConnection.doOneMessage(ServerConnection.java:920)
> at org.apache.geode.internal.cache.tier.sockets.ServerConnection.run(ServerConnection.java:1165)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at org.apache.geode.internal.cache.tier.sockets.AcceptorImpl$1$1.run(AcceptorImpl.java:579)
> at java.lang.Thread.run(Thread.java:745)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.20.7#820007)