You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ignite.apache.org by "Semyon Danilov (Jira)" <ji...@apache.org> on 2022/06/14 10:47:00 UTC
[jira] [Updated] (IGNITE-17159) Server node failed due to java.lang.AssertionError: Client already created
[ https://issues.apache.org/jira/browse/IGNITE-17159?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Semyon Danilov updated IGNITE-17159:
------------------------------------
Description:
It seems like we release recovery descriptor prior to removing communication client from the connection pool. So another thread successfully reserves descriptor, creates a client and tries to put a newly created client into the pool and fails because there is a stale client which we didn’t remove yet. I think we should release descriptor AFTER we remove communication client and it should fix the issue.
{noformat}
at org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool.addNodeClient(ConnectionClientPool.java:638)
at org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool.reserveClient(ConnectionClientPool.java:242)
at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:1174)
at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:1123)
at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1817)
at org.apache.ignite.internal.managers.communication.GridIoManager.sendToGridTopic(GridIoManager.java:1944)
at org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1265)
at org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1304)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture.sendDhtRequests(GridDhtAtomicAbstractUpdateFuture.java:489)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture.map(GridDhtAtomicAbstractUpdateFuture.java:445)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1921)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal(GridDhtAtomicCache.java:1685)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateFuture.sendSingleRequest(GridNearAtomicAbstractUpdateFuture.java:319)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture.map(GridNearAtomicSingleUpdateFuture.java:496)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture.mapOnTopology(GridNearAtomicSingleUpdateFuture.java:454)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateFuture.map(GridNearAtomicAbstractUpdateFuture.java:267)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.update0(GridDhtAtomicCache.java:1164)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.put0(GridDhtAtomicCache.java:627)
at org.apache.ignite.internal.processors.cache.GridCacheAdapter.put(GridCacheAdapter.java:2073)
at org.apache.ignite.internal.processors.cache.GridCacheAdapter.put(GridCacheAdapter.java:2048)
at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.put(IgniteCacheProxyImpl.java:1311)
at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.put(GatewayProtectedCacheProxy.java:817)
at com.autozone.supplychain.csr.receiver.QuantityCacheTupleReceiver.processRecord(QuantityCacheTupleReceiver.java:123)
at com.autozone.supplychain.csr.receiver.QuantityCacheTupleReceiver.receive(QuantityCacheTupleReceiver.java:47)
at org.apache.ignite.internal.processors.datastreamer.DataStreamerUpdateJob.call(DataStreamerUpdateJob.java:137)
at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.localUpdate(DataStreamProcessor.java:401)
at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.processRequest(DataStreamProcessor.java:306)
at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.access$000(DataStreamProcessor.java:59)
at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor$1.onMessage(DataStreamProcessor.java:89)
at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1727)
at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1334)
at org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:158)
at org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1218)
at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54)
at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:567)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119)
at java.lang.Thread.run(Thread.java:750)
{noformat}
was:
{noformat}
at org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool.addNodeClient(ConnectionClientPool.java:638)
at org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool.reserveClient(ConnectionClientPool.java:242)
at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:1174)
at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:1123)
at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1817)
at org.apache.ignite.internal.managers.communication.GridIoManager.sendToGridTopic(GridIoManager.java:1944)
at org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1265)
at org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1304)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture.sendDhtRequests(GridDhtAtomicAbstractUpdateFuture.java:489)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture.map(GridDhtAtomicAbstractUpdateFuture.java:445)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1921)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal(GridDhtAtomicCache.java:1685)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateFuture.sendSingleRequest(GridNearAtomicAbstractUpdateFuture.java:319)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture.map(GridNearAtomicSingleUpdateFuture.java:496)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture.mapOnTopology(GridNearAtomicSingleUpdateFuture.java:454)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateFuture.map(GridNearAtomicAbstractUpdateFuture.java:267)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.update0(GridDhtAtomicCache.java:1164)
at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.put0(GridDhtAtomicCache.java:627)
at org.apache.ignite.internal.processors.cache.GridCacheAdapter.put(GridCacheAdapter.java:2073)
at org.apache.ignite.internal.processors.cache.GridCacheAdapter.put(GridCacheAdapter.java:2048)
at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.put(IgniteCacheProxyImpl.java:1311)
at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.put(GatewayProtectedCacheProxy.java:817)
at com.autozone.supplychain.csr.receiver.QuantityCacheTupleReceiver.processRecord(QuantityCacheTupleReceiver.java:123)
at com.autozone.supplychain.csr.receiver.QuantityCacheTupleReceiver.receive(QuantityCacheTupleReceiver.java:47)
at org.apache.ignite.internal.processors.datastreamer.DataStreamerUpdateJob.call(DataStreamerUpdateJob.java:137)
at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.localUpdate(DataStreamProcessor.java:401)
at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.processRequest(DataStreamProcessor.java:306)
at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.access$000(DataStreamProcessor.java:59)
at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor$1.onMessage(DataStreamProcessor.java:89)
at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1727)
at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1334)
at org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:158)
at org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1218)
at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54)
at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:567)
at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119)
at java.lang.Thread.run(Thread.java:750)
{noformat}
> Server node failed due to java.lang.AssertionError: Client already created
> --------------------------------------------------------------------------
>
> Key: IGNITE-17159
> URL: https://issues.apache.org/jira/browse/IGNITE-17159
> Project: Ignite
> Issue Type: Bug
> Components: networking
> Reporter: Semyon Danilov
> Assignee: Semyon Danilov
> Priority: Major
>
> It seems like we release recovery descriptor prior to removing communication client from the connection pool. So another thread successfully reserves descriptor, creates a client and tries to put a newly created client into the pool and fails because there is a stale client which we didn’t remove yet. I think we should release descriptor AFTER we remove communication client and it should fix the issue.
> {noformat}
> at org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool.addNodeClient(ConnectionClientPool.java:638)
> at org.apache.ignite.spi.communication.tcp.internal.ConnectionClientPool.reserveClient(ConnectionClientPool.java:242)
> at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage0(TcpCommunicationSpi.java:1174)
> at org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi.sendMessage(TcpCommunicationSpi.java:1123)
> at org.apache.ignite.internal.managers.communication.GridIoManager.send(GridIoManager.java:1817)
> at org.apache.ignite.internal.managers.communication.GridIoManager.sendToGridTopic(GridIoManager.java:1944)
> at org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1265)
> at org.apache.ignite.internal.processors.cache.GridCacheIoManager.send(GridCacheIoManager.java:1304)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture.sendDhtRequests(GridDhtAtomicAbstractUpdateFuture.java:489)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture.map(GridDhtAtomicAbstractUpdateFuture.java:445)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal0(GridDhtAtomicCache.java:1921)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.updateAllAsyncInternal(GridDhtAtomicCache.java:1685)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateFuture.sendSingleRequest(GridNearAtomicAbstractUpdateFuture.java:319)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture.map(GridNearAtomicSingleUpdateFuture.java:496)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicSingleUpdateFuture.mapOnTopology(GridNearAtomicSingleUpdateFuture.java:454)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateFuture.map(GridNearAtomicAbstractUpdateFuture.java:267)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.update0(GridDhtAtomicCache.java:1164)
> at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.put0(GridDhtAtomicCache.java:627)
> at org.apache.ignite.internal.processors.cache.GridCacheAdapter.put(GridCacheAdapter.java:2073)
> at org.apache.ignite.internal.processors.cache.GridCacheAdapter.put(GridCacheAdapter.java:2048)
> at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.put(IgniteCacheProxyImpl.java:1311)
> at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.put(GatewayProtectedCacheProxy.java:817)
> at com.autozone.supplychain.csr.receiver.QuantityCacheTupleReceiver.processRecord(QuantityCacheTupleReceiver.java:123)
> at com.autozone.supplychain.csr.receiver.QuantityCacheTupleReceiver.receive(QuantityCacheTupleReceiver.java:47)
> at org.apache.ignite.internal.processors.datastreamer.DataStreamerUpdateJob.call(DataStreamerUpdateJob.java:137)
> at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.localUpdate(DataStreamProcessor.java:401)
> at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.processRequest(DataStreamProcessor.java:306)
> at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor.access$000(DataStreamProcessor.java:59)
> at org.apache.ignite.internal.processors.datastreamer.DataStreamProcessor$1.onMessage(DataStreamProcessor.java:89)
> at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1727)
> at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1334)
> at org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:158)
> at org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1218)
> at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54)
> at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:567)
> at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119)
> at java.lang.Thread.run(Thread.java:750)
> {noformat}
--
This message was sent by Atlassian Jira
(v8.20.7#820007)