You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@ignite.apache.org by "Grigory Domozhirov (Jira)" <ji...@apache.org> on 2020/07/30 12:11:00 UTC

[jira] [Updated] (IGNITE-13313) IndexOutOfBoundsException from GridDhtAtomicUpdateRequest on server node startup

     [ https://issues.apache.org/jira/browse/IGNITE-13313?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Grigory Domozhirov updated IGNITE-13313:
----------------------------------------
    Description: 
Sometimes server node fails on startup with following exception.

GGCE 8.7.21, no persistance, atomic caches

 
 2020-07-30 13:39:02,962 [sys-stripe-0-#1|#1] ERROR o.a.i.i.p.c.GridCacheIoManager - Failed processing message [senderId=98dc3c18-ea57-4805-9603-b92eb7e62be2, msg=GridD
 htAtomicUpdateRequest [keys=ArrayList [com.moex.esb.blackhole.model.fx.trade.Securities$Key [idHash=4701836, hash=890931906, secboard=AETS, seccode=GLDRUB_TOD],
 *_<skipped>_*
 ... and 111883 skipped ...=0,_*<skipped>*_
  prevVals=ArrayList [null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
  null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, n
 ull, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, nul
 l, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null... and 19 more], ttls=null, conflict
 ExpireTimes=null, nearTtls=null, nearExpireTimes=null, nearKeys=null, nearVals=null, obsoleteIndexes=null, forceTransformBackups=false, updateCntrs=GridLongList [id
 x=174, arr=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1,
 1,2,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,2,2,1,1,1,1,2,1,1,3,1,1,1,1,1,2,1,2,1,1,1,1,2,1,1,1,2,1,2,1,2,1,1,1,2,1,2,2,1,4,1,3,3,1,2,2,2,2,1,3,1,1,1,1,1,2,3,4,1,1,2,1,1,1,
 1,4,1,1,1,2,1,2,1,1,1,1,3,1,1,1]], super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=ae4abad0-d501-4703-98bf-b5eabd10f159, nearFutId=147459, flags=k
 eepBinary|hasRes]]]
 java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119
         at java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) ~[na:na]
         at java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) ~[na:na]
         at java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) ~[na:na]
         at java.base/java.util.Objects.checkIndex(Objects.java:373) ~[na:na]
         at java.base/java.util.ArrayList.get(ArrayList.java:425) ~[na:na]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateRequest.previousValue(GridDhtAtomicUpdateRequest.java:391) ~[ignite
 -core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3363) ~[ignit
 e-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:141) ~[ignite-core-8.7.21.jar:8.
 7.21]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:311) ~[ignite-core-8.7.21.jar:8.7.2
 1]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:306) ~[ignite-core-8.7.21.jar:8.7.2
 1]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1711) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1331) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:153) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1216) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:559) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) ~[ignite-core-8.7.21.jar:8.7.21]
         at java.base/java.lang.Thread.run(Thread.java:830) ~[na:na] 
  
  
 2020-07-30 13:39:02,964 [sys-stripe-0-#1|#1] ERROR ROOT - Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureH
 andler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], fa
 ilureCtx=FailureContext [type=CRITICAL_ERROR, err=java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119]]
 java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119
         at java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) ~[na:na]
         at java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) ~[na:na]
         at java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) ~[na:na]
         at java.base/java.util.Objects.checkIndex(Objects.java:373) ~[na:na]
         at java.base/java.util.ArrayList.get(ArrayList.java:425) ~[na:na]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateRequest.previousValue(GridDhtAtomicUpdateRequest.java:391) ~[ignite
 -core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3363) ~[ignit
 e-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:141) ~[ignite-core-8.7.21.jar:8.
 7.21]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:311) ~[ignite-core-8.7.21.jar:8.7.2
 1]
         at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:306) ~[ignite-core-8.7.21.jar:8.7.2
 1]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1711) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1331) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:153) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1216) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:559) ~[ignite-core-8.7.21.jar:8.7.21]
         at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) ~[ignite-core-8.7.21.jar:8.7.21]
         at java.base/java.lang.Thread.run(Thread.java:830) ~[na:na]

!https://owa.moex.com/owa/service.svc/s/GetFileAttachment?id=AQMkADI3MWRlYTcxLTk4NTItNDE1Mi04NzhhLTFlNmZlNGJjODBkZgBGAAADA0VgfW9PI06bbEcYGiv%2FdAcActmJFw3L1U%2ByDGn9%2B1yz%2FAAAAgEPAAAActmJFw3L1U%2ByDGn9%2B1yz%2FAACFYvp3gAAAAESABAAwxeg969M%2Fk2HFngHONZ4zA%3D%3D&X-OWA-CANARY=LvEoq6xN8UeyuWHAH2kxHwDiVHXQJNgIWDr-DG2WSjSwvZaJYt6CkBV-j6eaZGO0K24etDWd7Oo.!

  was:
Sometimes server node fails on startup with following exception.

GGCE 8.7.21, no persistance, atomic caches

 
2020-07-30 13:39:02,962 [sys-stripe-0-#1] ERROR o.a.i.i.p.c.GridCacheIoManager - Failed processing message [senderId=98dc3c18-ea57-4805-9603-b92eb7e62be2, msg=GridD
htAtomicUpdateRequest [keys=ArrayList [com.moex.esb.blackhole.model.fx.trade.Securities$Key [idHash=4701836, hash=890931906, secboard=AETS, seccode=GLDRUB_TOD],
*_<skipped>_*
... and 111883 skipped ...=0,_*<skipped>*_
 prevVals=ArrayList [null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
 null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, n
ull, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, nul
l, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null... and 19 more], ttls=null, conflict
ExpireTimes=null, nearTtls=null, nearExpireTimes=null, nearKeys=null, nearVals=null, obsoleteIndexes=null, forceTransformBackups=false, updateCntrs=GridLongList [id
x=174, arr=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1,
1,2,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,2,2,1,1,1,1,2,1,1,3,1,1,1,1,1,2,1,2,1,1,1,1,2,1,1,1,2,1,2,1,2,1,1,1,2,1,2,2,1,4,1,3,3,1,2,2,2,2,1,3,1,1,1,1,1,2,3,4,1,1,2,1,1,1,
1,4,1,1,1,2,1,2,1,1,1,1,3,1,1,1]], super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=ae4abad0-d501-4703-98bf-b5eabd10f159, nearFutId=147459, flags=k
eepBinary|hasRes]]]
java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119
        at java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) ~[na:na]
        at java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) ~[na:na]
        at java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) ~[na:na]
        at java.base/java.util.Objects.checkIndex(Objects.java:373) ~[na:na]
        at java.base/java.util.ArrayList.get(ArrayList.java:425) ~[na:na]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateRequest.previousValue(GridDhtAtomicUpdateRequest.java:391) ~[ignite
-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3363) ~[ignit
e-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:141) ~[ignite-core-8.7.21.jar:8.
7.21]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:311) ~[ignite-core-8.7.21.jar:8.7.2
1]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:306) ~[ignite-core-8.7.21.jar:8.7.2
1]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1711) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1331) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:153) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1216) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:559) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) ~[ignite-core-8.7.21.jar:8.7.21]
        at java.base/java.lang.Thread.run(Thread.java:830) ~[na:na] 
 
 
2020-07-30 13:39:02,964 [sys-stripe-0-#1] ERROR ROOT - Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureH
andler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], fa
ilureCtx=FailureContext [type=CRITICAL_ERROR, err=java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119]]
java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119
        at java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) ~[na:na]
        at java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) ~[na:na]
        at java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) ~[na:na]
        at java.base/java.util.Objects.checkIndex(Objects.java:373) ~[na:na]
        at java.base/java.util.ArrayList.get(ArrayList.java:425) ~[na:na]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateRequest.previousValue(GridDhtAtomicUpdateRequest.java:391) ~[ignite
-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3363) ~[ignit
e-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:141) ~[ignite-core-8.7.21.jar:8.
7.21]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:311) ~[ignite-core-8.7.21.jar:8.7.2
1]
        at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:306) ~[ignite-core-8.7.21.jar:8.7.2
1]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1711) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1331) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:153) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1216) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:559) ~[ignite-core-8.7.21.jar:8.7.21]
        at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) ~[ignite-core-8.7.21.jar:8.7.21]
        at java.base/java.lang.Thread.run(Thread.java:830) ~[na:na]
 
Управление разработки интеграционных сервисов

Рабочий +7 (495) 363-32-32, доб 3190

!https://owa.moex.com/owa/service.svc/s/GetFileAttachment?id=AQMkADI3MWRlYTcxLTk4NTItNDE1Mi04NzhhLTFlNmZlNGJjODBkZgBGAAADA0VgfW9PI06bbEcYGiv%2FdAcActmJFw3L1U%2ByDGn9%2B1yz%2FAAAAgEPAAAActmJFw3L1U%2ByDGn9%2B1yz%2FAACFYvp3gAAAAESABAAwxeg969M%2Fk2HFngHONZ4zA%3D%3D&X-OWA-CANARY=LvEoq6xN8UeyuWHAH2kxHwDiVHXQJNgIWDr-DG2WSjSwvZaJYt6CkBV-j6eaZGO0K24etDWd7Oo.!


> IndexOutOfBoundsException from GridDhtAtomicUpdateRequest on server node startup
> --------------------------------------------------------------------------------
>
>                 Key: IGNITE-13313
>                 URL: https://issues.apache.org/jira/browse/IGNITE-13313
>             Project: Ignite
>          Issue Type: Bug
>            Reporter: Grigory Domozhirov
>            Priority: Major
>
> Sometimes server node fails on startup with following exception.
> GGCE 8.7.21, no persistance, atomic caches
>  
>  2020-07-30 13:39:02,962 [sys-stripe-0-#1|#1] ERROR o.a.i.i.p.c.GridCacheIoManager - Failed processing message [senderId=98dc3c18-ea57-4805-9603-b92eb7e62be2, msg=GridD
>  htAtomicUpdateRequest [keys=ArrayList [com.moex.esb.blackhole.model.fx.trade.Securities$Key [idHash=4701836, hash=890931906, secboard=AETS, seccode=GLDRUB_TOD],
>  *_<skipped>_*
>  ... and 111883 skipped ...=0,_*<skipped>*_
>   prevVals=ArrayList [null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
>   null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, n
>  ull, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, nul
>  l, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null... and 19 more], ttls=null, conflict
>  ExpireTimes=null, nearTtls=null, nearExpireTimes=null, nearKeys=null, nearVals=null, obsoleteIndexes=null, forceTransformBackups=false, updateCntrs=GridLongList [id
>  x=174, arr=[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,2,1,1,
>  1,2,1,1,1,1,1,1,3,1,1,1,1,1,1,1,1,1,2,2,1,1,1,1,2,1,1,3,1,1,1,1,1,2,1,2,1,1,1,1,2,1,1,1,2,1,2,1,2,1,1,1,2,1,2,2,1,4,1,3,3,1,2,2,2,2,1,3,1,1,1,1,1,2,3,4,1,1,2,1,1,1,
>  1,4,1,1,1,2,1,2,1,1,1,1,3,1,1,1]], super=GridDhtAtomicAbstractUpdateRequest [onRes=false, nearNodeId=ae4abad0-d501-4703-98bf-b5eabd10f159, nearFutId=147459, flags=k
>  eepBinary|hasRes]]]
>  java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119
>          at java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) ~[na:na]
>          at java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) ~[na:na]
>          at java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) ~[na:na]
>          at java.base/java.util.Objects.checkIndex(Objects.java:373) ~[na:na]
>          at java.base/java.util.ArrayList.get(ArrayList.java:425) ~[na:na]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateRequest.previousValue(GridDhtAtomicUpdateRequest.java:391) ~[ignite
>  -core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3363) ~[ignit
>  e-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:141) ~[ignite-core-8.7.21.jar:8.
>  7.21]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:311) ~[ignite-core-8.7.21.jar:8.7.2
>  1]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:306) ~[ignite-core-8.7.21.jar:8.7.2
>  1]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1711) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1331) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:153) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1216) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:559) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) ~[ignite-core-8.7.21.jar:8.7.21]
>          at java.base/java.lang.Thread.run(Thread.java:830) ~[na:na] 
>   
>   
>  2020-07-30 13:39:02,964 [sys-stripe-0-#1|#1] ERROR ROOT - Critical system error detected. Will be handled accordingly to configured handler [hnd=StopNodeOrHaltFailureH
>  andler [tryStop=false, timeout=0, super=AbstractFailureHandler [ignoredFailureTypes=UnmodifiableSet [SYSTEM_WORKER_BLOCKED, SYSTEM_CRITICAL_OPERATION_TIMEOUT]]], fa
>  ilureCtx=FailureContext [type=CRITICAL_ERROR, err=java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119]]
>  java.lang.IndexOutOfBoundsException: Index 119 out of bounds for length 119
>          at java.base/jdk.internal.util.Preconditions.outOfBounds(Preconditions.java:64) ~[na:na]
>          at java.base/jdk.internal.util.Preconditions.outOfBoundsCheckIndex(Preconditions.java:70) ~[na:na]
>          at java.base/jdk.internal.util.Preconditions.checkIndex(Preconditions.java:248) ~[na:na]
>          at java.base/java.util.Objects.checkIndex(Objects.java:373) ~[na:na]
>          at java.base/java.util.ArrayList.get(ArrayList.java:425) ~[na:na]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicUpdateRequest.previousValue(GridDhtAtomicUpdateRequest.java:391) ~[ignite
>  -core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.processDhtAtomicUpdateRequest(GridDhtAtomicCache.java:3363) ~[ignit
>  e-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache.access$600(GridDhtAtomicCache.java:141) ~[ignite-core-8.7.21.jar:8.
>  7.21]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:311) ~[ignite-core-8.7.21.jar:8.7.2
>  1]
>          at org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicCache$7.apply(GridDhtAtomicCache.java:306) ~[ignite-core-8.7.21.jar:8.7.2
>  1]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1142) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:591) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:392) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.handleMessage(GridCacheIoManager.java:318) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$100(GridCacheIoManager.java:109) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.processors.cache.GridCacheIoManager$1.onMessage(GridCacheIoManager.java:308) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1711) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.GridIoManager.processRegularMessage0(GridIoManager.java:1331) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.GridIoManager.access$4800(GridIoManager.java:153) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.GridIoManager$8.execute(GridIoManager.java:1216) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.managers.communication.TraceRunnable.run(TraceRunnable.java:54) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.util.StripedExecutor$Stripe.body(StripedExecutor.java:559) ~[ignite-core-8.7.21.jar:8.7.21]
>          at org.apache.ignite.internal.util.worker.GridWorker.run(GridWorker.java:119) ~[ignite-core-8.7.21.jar:8.7.21]
>          at java.base/java.lang.Thread.run(Thread.java:830) ~[na:na]
> !https://owa.moex.com/owa/service.svc/s/GetFileAttachment?id=AQMkADI3MWRlYTcxLTk4NTItNDE1Mi04NzhhLTFlNmZlNGJjODBkZgBGAAADA0VgfW9PI06bbEcYGiv%2FdAcActmJFw3L1U%2ByDGn9%2B1yz%2FAAAAgEPAAAActmJFw3L1U%2ByDGn9%2B1yz%2FAACFYvp3gAAAAESABAAwxeg969M%2Fk2HFngHONZ4zA%3D%3D&X-OWA-CANARY=LvEoq6xN8UeyuWHAH2kxHwDiVHXQJNgIWDr-DG2WSjSwvZaJYt6CkBV-j6eaZGO0K24etDWd7Oo.!



--
This message was sent by Atlassian Jira
(v8.3.4#803005)