You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@ignite.apache.org by ihalilaltun <ib...@segmentify.com> on 2019/07/17 10:55:20 UTC
NegativeArraySizeException on cluster rebalance
Hi Igniters,
We are getting negativearraysizeexception on one of our cacheGroups
rebalancing period. I am adding the details that I could get from logs;
*Ignite version*: 2.7.5
*Cluster size*: 16
*Client size*: 22
*Cluster OS version*: Centos 7
*Cluster Kernel version*: 4.4.185-1.el7.elrepo.x86_64
*Java version* :
java version "1.8.0_201"
Java(TM) SE Runtime Environment (build 1.8.0_201-b09)
Java HotSpot(TM) 64-Bit Server VM (build 25.201-b09, mixed mode)
Pojo we use;
UserPriceDropRecord.txt
<http://apache-ignite-users.70518.x6.nabble.com/file/t2515/UserPriceDropRecord.txt>
Error we got;
[2019-07-17T10:23:13,162][ERROR][sys-#241][GridDhtPartitionSupplier] Failed
to continue supplying [grp=userPriceDropDataCacheGroup,
demander=12d8bad8-62a9-465d-aca4-4afa203d6778,
topVer=AffinityTopologyVersion [topVer=238, minorTopVer=0], topic=1]
java.lang.NegativeArraySizeException: null
at org.apache.ignite.internal.pagemem.PageUtils.getBytes(PageUtils.java:63)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.readFullRow(CacheDataRowAdapter.java:330)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:167)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:108)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.tree.DataRow.<init>(DataRow.java:55)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:92)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:200)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:49)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$ForwardCursor.fillFromBuffer0(BPlusTree.java:5512)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.fillFromBuffer(BPlusTree.java:5280)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.nextPage(BPlusTree.java:5332)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$ForwardCursor.next(BPlusTree.java:5566)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$9.onHasNext(IgniteCacheOffheapManagerImpl.java:1185)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:53)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.advance(IgniteRebalanceIteratorImpl.java:79)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.nextX(IgniteRebalanceIteratorImpl.java:139)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:185)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:37)
~[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplier.handleDemandMessage(GridDhtPartitionSupplier.java:333)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.handleDemandMessage(GridDhtPreloader.java:404)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$5.apply(GridCachePartitionExchangeManager.java:424)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$5.apply(GridCachePartitionExchangeManager.java:409)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1056)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:581)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$700(GridCacheIoManager.java:101)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.processors.cache.GridCacheIoManager$OrderedMessageListener.onMessage(GridCacheIoManager.java:1613)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1569)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$4100(GridIoManager.java:127)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.managers.communication.GridIoManager$GridCommunicationMessageSet.unwind(GridIoManager.java:2768)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.managers.communication.GridIoManager.unwindMessageSet(GridIoManager.java:1529)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.managers.communication.GridIoManager.access$4400(GridIoManager.java:127)
[ignite-core-2.7.5.jar:2.7.5]
at
org.apache.ignite.internal.managers.communication.GridIoManager$10.run(GridIoManager.java:1498)
[ignite-core-2.7.5.jar:2.7.5]
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[?:1.8.0_201]
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[?:1.8.0_201]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_201]
-----
İbrahim Halil Altun
Senior Software Engineer @ Segmentify
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: NegativeArraySizeException on cluster rebalance
Posted by Dmitriy Govorukhin <dm...@gmail.com>.
Hi Ibrahim,
Looks like Btree corrupted problem, I guess it was fixed in
https://issues.apache.org/jira/browse/IGNITE-11953
The fix you can find in 9c323149db1cee0ff6586389def059a85428b116
org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java:1631
Thank you!
On Wed, Jul 17, 2019 at 1:55 PM ihalilaltun <ib...@segmentify.com>
wrote:
> Hi Igniters,
>
> We are getting negativearraysizeexception on one of our cacheGroups
> rebalancing period. I am adding the details that I could get from logs;
>
> *Ignite version*: 2.7.5
> *Cluster size*: 16
> *Client size*: 22
> *Cluster OS version*: Centos 7
> *Cluster Kernel version*: 4.4.185-1.el7.elrepo.x86_64
> *Java version* :
> java version "1.8.0_201"
> Java(TM) SE Runtime Environment (build 1.8.0_201-b09)
> Java HotSpot(TM) 64-Bit Server VM (build 25.201-b09, mixed mode)
>
>
> Pojo we use;
> UserPriceDropRecord.txt
> <
> http://apache-ignite-users.70518.x6.nabble.com/file/t2515/UserPriceDropRecord.txt>
>
>
> Error we got;
>
> [2019-07-17T10:23:13,162][ERROR][sys-#241][GridDhtPartitionSupplier] Failed
> to continue supplying [grp=userPriceDropDataCacheGroup,
> demander=12d8bad8-62a9-465d-aca4-4afa203d6778,
> topVer=AffinityTopologyVersion [topVer=238, minorTopVer=0], topic=1]
> java.lang.NegativeArraySizeException: null
> at
> org.apache.ignite.internal.pagemem.PageUtils.getBytes(PageUtils.java:63)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.readFullRow(CacheDataRowAdapter.java:330)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:167)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter.initFromLink(CacheDataRowAdapter.java:108)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.tree.DataRow.<init>(DataRow.java:55)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.tree.CacheDataRowStore.dataRow(CacheDataRowStore.java:92)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:200)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.tree.CacheDataTree.getRow(CacheDataTree.java:49)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$ForwardCursor.fillFromBuffer0(BPlusTree.java:5512)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.fillFromBuffer(BPlusTree.java:5280)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$AbstractForwardCursor.nextPage(BPlusTree.java:5332)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree$ForwardCursor.next(BPlusTree.java:5566)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl$9.onHasNext(IgniteCacheOffheapManagerImpl.java:1185)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.util.GridCloseableIteratorAdapter.hasNextX(GridCloseableIteratorAdapter.java:53)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.advance(IgniteRebalanceIteratorImpl.java:79)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.nextX(IgniteRebalanceIteratorImpl.java:139)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:185)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.IgniteRebalanceIteratorImpl.next(IgniteRebalanceIteratorImpl.java:37)
> ~[ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionSupplier.handleDemandMessage(GridDhtPartitionSupplier.java:333)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader.handleDemandMessage(GridDhtPreloader.java:404)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$5.apply(GridCachePartitionExchangeManager.java:424)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager$5.apply(GridCachePartitionExchangeManager.java:409)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.processMessage(GridCacheIoManager.java:1056)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.onMessage0(GridCacheIoManager.java:581)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.GridCacheIoManager.access$700(GridCacheIoManager.java:101)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.processors.cache.GridCacheIoManager$OrderedMessageListener.onMessage(GridCacheIoManager.java:1613)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.managers.communication.GridIoManager.invokeListener(GridIoManager.java:1569)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.managers.communication.GridIoManager.access$4100(GridIoManager.java:127)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.managers.communication.GridIoManager$GridCommunicationMessageSet.unwind(GridIoManager.java:2768)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.managers.communication.GridIoManager.unwindMessageSet(GridIoManager.java:1529)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.managers.communication.GridIoManager.access$4400(GridIoManager.java:127)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> org.apache.ignite.internal.managers.communication.GridIoManager$10.run(GridIoManager.java:1498)
> [ignite-core-2.7.5.jar:2.7.5]
> at
>
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> [?:1.8.0_201]
> at
>
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> [?:1.8.0_201]
> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_201]
>
>
>
>
> -----
> İbrahim Halil Altun
> Senior Software Engineer @ Segmentify
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>