You are viewing a plain text version of this content. The canonical link for it is here.
Posted to user@ignite.apache.org by ignite_user2016 <ri...@gmail.com> on 2017/09/01 17:48:46 UTC
Ignite 2.0 out of memory
Hello igniters,
I see the following warning in the log -
[17:23:29,070][WARN ][main][TcpCommunicationSpi] Message queue limit is set
to 0 which may lead to potential OOMEs when running cache operations in
FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and
receiver sides.
Is this warning relevant to following error ?
And then our server went down with following error -
java.lang.OutOfMemoryError: unable to create new native thread
at java.lang.Thread.start0(Native Method) ~[?:1.8.0_121]
at java.lang.Thread.start(Thread.java:714) ~[?:1.8.0_121]
at
org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(GridDiscoveryManager.java:726)
~[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.internal.IgniteKernal.startManager(IgniteKernal.java:1745)
~[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:968)
[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1895)
[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1647)
[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1075)
[ignite-core-2.0.0.jar:2.0.0]
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:595)
[ignite-core-2.0.0.jar:2.0.0]
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:519)
[ignite-core-2.0.0.jar:2.0.0]
at org.apache.ignite.Ignition.start(Ignition.java:322)
[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.open.VisorOpenCommand.open(VisorOpenCommand.scala:251)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.open.VisorOpenCommand.open(VisorOpenCommand.scala:219)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.open.VisorOpenCommand$$anonfun$2.apply(VisorOpenCommand.scala:306)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.open.VisorOpenCommand$$anonfun$2.apply(VisorOpenCommand.scala:306)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.VisorConsole.mainLoop(VisorConsole.scala:217)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.VisorConsole$.delayedEndpoint$org$apache$ignite$visor$commands$VisorConsole$1(VisorConsole.scala:329)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.VisorConsole$delayedInit$body.apply(VisorConsole.scala:318)
[ignite-visor-console-2.0.0.jar:2.0.0]
at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
[scala-library-2.11.7.jar:?]
at
scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
[scala-library-2.11.7.jar:?]
at scala.App$$anonfun$main$1.apply(App.scala:76)
[scala-library-2.11.7.jar:?]
at scala.App$$anonfun$main$1.apply(App.scala:76)
[scala-library-2.11.7.jar:?]
at scala.collection.immutable.List.foreach(List.scala:381)
[scala-library-2.11.7.jar:?]
at
scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:35)
[scala-library-2.11.7.jar:?]
at scala.App$class.main(App.scala:76) [scala-library-2.11.7.jar:?]
at
org.apache.ignite.visor.commands.VisorConsole$.main(VisorConsole.scala:318)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.VisorConsole.main(VisorConsole.scala)
[ignite-visor-console-2.0.0.jar:2.0.0]
[22:25:28,566][ERROR][main][G] Failed to initialize striped pool.
java.lang.OutOfMemoryError: unable to create new native thread
at java.lang.Thread.start0(Native Method) ~[?:1.8.0_121]
at java.lang.Thread.start(Thread.java:714) ~[?:1.8.0_121]
at
org.apache.ignite.internal.util.StripedExecutor$Stripe.start(StripedExecutor.java:444)
~[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.internal.util.StripedExecutor.<init>(StripedExecutor.java:84)
[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(IgnitionEx.java:1729)
[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(IgnitionEx.java:1647)
[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1075)
[ignite-core-2.0.0.jar:2.0.0]
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:595)
[ignite-core-2.0.0.jar:2.0.0]
at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.java:519)
[ignite-core-2.0.0.jar:2.0.0]
at org.apache.ignite.Ignition.start(Ignition.java:322)
[ignite-core-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.open.VisorOpenCommand.open(VisorOpenCommand.scala:251)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.open.VisorOpenCommand.open(VisorOpenCommand.scala:219)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.open.VisorOpenCommand$$anonfun$2.apply(VisorOpenCommand.scala:306)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.open.VisorOpenCommand$$anonfun$2.apply(VisorOpenCommand.scala:306)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.VisorConsole.mainLoop(VisorConsole.scala:217)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.VisorConsole$.delayedEndpoint$org$apache$ignite$visor$commands$VisorConsole$1(VisorConsole.scala:329)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.VisorConsole$delayedInit$body.apply(VisorConsole.scala:318)
[ignite-visor-console-2.0.0.jar:2.0.0]
at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
[scala-library-2.11.7.jar:?]
at
scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
[scala-library-2.11.7.jar:?]
at scala.App$$anonfun$main$1.apply(App.scala:76)
[scala-library-2.11.7.jar:?]
at scala.App$$anonfun$main$1.apply(App.scala:76)
[scala-library-2.11.7.jar:?]
at scala.collection.immutable.List.foreach(List.scala:381)
[scala-library-2.11.7.jar:?]
at
scala.collection.generic.TraversableForwarder$class.foreach(TraversableForwarder.scala:35)
[scala-library-2.11.7.jar:?]
at scala.App$class.main(App.scala:76) [scala-library-2.11.7.jar:?]
at
org.apache.ignite.visor.commands.VisorConsole$.main(VisorConsole.scala:318)
[ignite-visor-console-2.0.0.jar:2.0.0]
at
org.apache.ignite.visor.commands.VisorConsole.main(VisorConsole.scala)
[ignite-visor-console-2.0.0.jar:2.0.0]
we run Ignite with bare minimum configuration and mostly we use it for
spring cache.
And we also use visor for monitoring ignite in our prod environment.
any help will be much appreciated here ..
Thanks ..
Rishi
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Ignite 2.0 out of memory
Posted by ignite_user2016 <ri...@gmail.com>.
Thanks for the quick response.
how many nodes you have on the one machine? we have single ignite node on
the machine.
How many visor clients on it machine? we mostly monitor ignite using
ignitevisor, and it might have happened the visor client did not shut down
correctly.
I think, it could be an issue with Nagios monitoring so will see if run into
same issue again however will try to get the logs from ops team.
Thanks ..
--
Sent from: http://apache-ignite-users.70518.x6.nabble.com/
Re: Ignite 2.0 out of memory
Posted by Evgenii Zhuravlev <e....@gmail.com>.
This error is not about heap memory, it indicates that you started more
threads within one process that it's allowed by OS. On Linux you can play
with ulimit to overcome the issue.
Also, how many nodes you have on the one machine? How many visor clients on
it machine? Could you share logs from nodes? Could it be possible that you
run more than one visor and just forget it, because it's not shown in
topology?
Evgenii
2017-09-01 20:48 GMT+03:00 ignite_user2016 <ri...@gmail.com>:
> Hello igniters,
>
> I see the following warning in the log -
>
> [17:23:29,070][WARN ][main][TcpCommunicationSpi] Message queue limit is set
> to 0 which may lead to potential OOMEs when running cache operations in
> FULL_ASYNC or PRIMARY_SYNC modes due to message queues growth on sender and
> receiver sides.
>
> Is this warning relevant to following error ?
>
> And then our server went down with following error -
>
> java.lang.OutOfMemoryError: unable to create new native thread
> at java.lang.Thread.start0(Native Method) ~[?:1.8.0_121]
> at java.lang.Thread.start(Thread.java:714) ~[?:1.8.0_121]
> at
> org.apache.ignite.internal.managers.discovery.GridDiscoveryManager.start(
> GridDiscoveryManager.java:726)
> ~[ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.internal.IgniteKernal.startManager(
> IgniteKernal.java:1745)
> ~[ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.internal.IgniteKernal.start(IgniteKernal.java:968)
> [ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(
> IgnitionEx.java:1895)
> [ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(
> IgnitionEx.java:1647)
> [ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1075)
> [ignite-core-2.0.0.jar:2.0.0]
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.
> java:595)
> [ignite-core-2.0.0.jar:2.0.0]
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.
> java:519)
> [ignite-core-2.0.0.jar:2.0.0]
> at org.apache.ignite.Ignition.start(Ignition.java:322)
> [ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.open.VisorOpenCommand.open(
> VisorOpenCommand.scala:251)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.open.VisorOpenCommand.open(
> VisorOpenCommand.scala:219)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.open.VisorOpenCommand$$anonfun$2.
> apply(VisorOpenCommand.scala:306)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.open.VisorOpenCommand$$anonfun$2.
> apply(VisorOpenCommand.scala:306)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.VisorConsole.mainLoop(VisorConsole.scala:
> 217)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.VisorConsole$.delayedEndpoint$org$apache$
> ignite$visor$commands$VisorConsole$1(VisorConsole.scala:329)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.VisorConsole$delayedInit$body.apply(
> VisorConsole.scala:318)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
> [scala-library-2.11.7.jar:?]
> at
> scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
> [scala-library-2.11.7.jar:?]
> at scala.App$$anonfun$main$1.apply(App.scala:76)
> [scala-library-2.11.7.jar:?]
> at scala.App$$anonfun$main$1.apply(App.scala:76)
> [scala-library-2.11.7.jar:?]
> at scala.collection.immutable.List.foreach(List.scala:381)
> [scala-library-2.11.7.jar:?]
> at
> scala.collection.generic.TraversableForwarder$class.
> foreach(TraversableForwarder.scala:35)
> [scala-library-2.11.7.jar:?]
> at scala.App$class.main(App.scala:76) [scala-library-2.11.7.jar:?]
> at
> org.apache.ignite.visor.commands.VisorConsole$.main(
> VisorConsole.scala:318)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.VisorConsole.main(VisorConsole.scala)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> [22:25:28,566][ERROR][main][G] Failed to initialize striped pool.
> java.lang.OutOfMemoryError: unable to create new native thread
> at java.lang.Thread.start0(Native Method) ~[?:1.8.0_121]
> at java.lang.Thread.start(Thread.java:714) ~[?:1.8.0_121]
> at
> org.apache.ignite.internal.util.StripedExecutor$Stripe.
> start(StripedExecutor.java:444)
> ~[ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.internal.util.StripedExecutor.<init>(
> StripedExecutor.java:84)
> [ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start0(
> IgnitionEx.java:1729)
> [ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.internal.IgnitionEx$IgniteNamedInstance.start(
> IgnitionEx.java:1647)
> [ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.internal.IgnitionEx.start0(IgnitionEx.java:1075)
> [ignite-core-2.0.0.jar:2.0.0]
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.
> java:595)
> [ignite-core-2.0.0.jar:2.0.0]
> at org.apache.ignite.internal.IgnitionEx.start(IgnitionEx.
> java:519)
> [ignite-core-2.0.0.jar:2.0.0]
> at org.apache.ignite.Ignition.start(Ignition.java:322)
> [ignite-core-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.open.VisorOpenCommand.open(
> VisorOpenCommand.scala:251)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.open.VisorOpenCommand.open(
> VisorOpenCommand.scala:219)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.open.VisorOpenCommand$$anonfun$2.
> apply(VisorOpenCommand.scala:306)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.open.VisorOpenCommand$$anonfun$2.
> apply(VisorOpenCommand.scala:306)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.VisorConsole.mainLoop(VisorConsole.scala:
> 217)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.VisorConsole$.delayedEndpoint$org$apache$
> ignite$visor$commands$VisorConsole$1(VisorConsole.scala:329)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.VisorConsole$delayedInit$body.apply(
> VisorConsole.scala:318)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at scala.Function0$class.apply$mcV$sp(Function0.scala:34)
> [scala-library-2.11.7.jar:?]
> at
> scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12)
> [scala-library-2.11.7.jar:?]
> at scala.App$$anonfun$main$1.apply(App.scala:76)
> [scala-library-2.11.7.jar:?]
> at scala.App$$anonfun$main$1.apply(App.scala:76)
> [scala-library-2.11.7.jar:?]
> at scala.collection.immutable.List.foreach(List.scala:381)
> [scala-library-2.11.7.jar:?]
> at
> scala.collection.generic.TraversableForwarder$class.
> foreach(TraversableForwarder.scala:35)
> [scala-library-2.11.7.jar:?]
> at scala.App$class.main(App.scala:76) [scala-library-2.11.7.jar:?]
> at
> org.apache.ignite.visor.commands.VisorConsole$.main(
> VisorConsole.scala:318)
> [ignite-visor-console-2.0.0.jar:2.0.0]
> at
> org.apache.ignite.visor.commands.VisorConsole.main(VisorConsole.scala)
> [ignite-visor-console-2.0.0.jar:2.0.0]
>
>
> we run Ignite with bare minimum configuration and mostly we use it for
> spring cache.
>
> And we also use visor for monitoring ignite in our prod environment.
>
> any help will be much appreciated here ..
>
> Thanks ..
>
> Rishi
>
>
>
>
>
>
>
> --
> Sent from: http://apache-ignite-users.70518.x6.nabble.com/
>