You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by mp...@apache.org on 2025/01/28 12:51:07 UTC
(ignite) branch master updated: IGNITE-23085 Enhanced error message in case node failed with OOM (#11502)
This is an automated email from the ASF dual-hosted git repository.
mpetrov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 3f0c010243f IGNITE-23085 Enhanced error message in case node failed with OOM (#11502)
3f0c010243f is described below
commit 3f0c010243f6ea7d84b35486f109ae02f1264110
Author: Maksim Davydov <70...@users.noreply.github.com>
AuthorDate: Tue Jan 28 15:50:59 2025 +0300
IGNITE-23085 Enhanced error message in case node failed with OOM (#11502)
---
.../org/apache/ignite/internal/IgniteKernal.java | 42 ++---------
.../processors/failure/FailureProcessor.java | 16 ++++-
.../apache/ignite/internal/util/IgniteUtils.java | 50 +++++++++++++
.../ignite/failure/OomFailureHandlerTest.java | 84 ++++++++++++++++++----
4 files changed, 137 insertions(+), 55 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 0973010aee3..00e3df82968 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -256,9 +256,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_US
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MARSHALLER_USE_DFLT_SUID;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MEMORY_CONFIG;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTENT_ID;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_OFFHEAP_SIZE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PEER_CLASSLOADING;
-import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PREFIX;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_REBALANCE_POOL_SIZE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_RESTART_ENABLED;
@@ -272,6 +270,7 @@ import static org.apache.ignite.internal.IgniteVersionUtils.BUILD_TSTAMP_STR;
import static org.apache.ignite.internal.IgniteVersionUtils.COPYRIGHT;
import static org.apache.ignite.internal.IgniteVersionUtils.VER;
import static org.apache.ignite.internal.IgniteVersionUtils.VER_STR;
+import static org.apache.ignite.internal.util.IgniteUtils.validateRamUsage;
import static org.apache.ignite.lifecycle.LifecycleEventType.AFTER_NODE_START;
import static org.apache.ignite.lifecycle.LifecycleEventType.BEFORE_NODE_START;
import static org.apache.ignite.mxbean.IgniteMXBean.ACTIVE_DESC;
@@ -1479,44 +1478,11 @@ public class IgniteKernal implements IgniteEx, Externalizable {
/**
* Checks whether physical RAM is not exceeded.
*/
- @SuppressWarnings("ConstantConditions")
private void checkPhysicalRam() {
- long ram = ctx.discovery().localNode().attribute(ATTR_PHY_RAM);
+ String validationResult = validateRamUsage(ctx);
- if (ram != -1) {
- String macs = ctx.discovery().localNode().attribute(ATTR_MACS);
-
- long totalHeap = 0;
- long totalOffheap = 0;
-
- for (ClusterNode node : ctx.discovery().allNodes()) {
- if (macs.equals(node.attribute(ATTR_MACS))) {
- long heap = node.metrics().getHeapMemoryMaximum();
- Long offheap = node.<Long>attribute(ATTR_OFFHEAP_SIZE);
-
- if (heap != -1)
- totalHeap += heap;
-
- if (offheap != null)
- totalOffheap += offheap;
- }
- }
-
- long total = totalHeap + totalOffheap;
-
- if (total < 0)
- total = Long.MAX_VALUE;
-
- // 4GB or 20% of available memory is expected to be used by OS and user applications
- long safeToUse = ram - Math.max(4L << 30, (long)(ram * 0.2));
-
- if (total > safeToUse) {
- U.quietAndWarn(log, "Nodes started on local machine require more than 80% of physical RAM what can " +
- "lead to significant slowdown due to swapping (please decrease JVM heap size, data region " +
- "size or checkpoint buffer size) [required=" + (total >> 20) + "MB, available=" +
- (ram >> 20) + "MB]");
- }
- }
+ if (validationResult != null)
+ U.quietAndWarn(log, validationResult);
}
/** */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
index 0876630dfea..b2fabbe4109 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/failure/FailureProcessor.java
@@ -40,6 +40,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_DUMP_THREADS_ON_FAILURE_THROTTLING_TIMEOUT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE;
+import static org.apache.ignite.internal.util.IgniteUtils.validateRamUsage;
/**
* General failure processing API
@@ -48,6 +49,10 @@ public class FailureProcessor extends GridProcessorAdapter {
/** @see IgniteSystemProperties#IGNITE_FAILURE_HANDLER_RESERVE_BUFFER_SIZE */
public static final int DFLT_FAILURE_HANDLER_RESERVE_BUFFER_SIZE = 64 * 1024;
+ /** OOM fix recommendation message. */
+ private static final String OOM_FIX_RECOMMENDATION_LOG_MSG = "Insufficient Java heap space. The following Ignite" +
+ " configuration problems have been detected that may caused the node failure: ";
+
/** Value of the system property that enables threads dumping on failure. */
private final boolean igniteDumpThreadsOnFailure =
IgniteSystemProperties.getBoolean(IGNITE_DUMP_THREADS_ON_FAILURE, true);
@@ -179,8 +184,15 @@ public class FailureProcessor extends GridProcessorAdapter {
"[hnd=" + hnd + ", failureCtx=" + failureCtx + ']', failureCtx.error());
}
- if (reserveBuf != null && X.hasCause(failureCtx.error(), OutOfMemoryError.class))
- reserveBuf = null;
+ if (X.hasCause(failureCtx.error(), OutOfMemoryError.class)) {
+ if (reserveBuf != null)
+ reserveBuf = null;
+
+ String validationResult = validateRamUsage(ctx);
+
+ if (validationResult != null)
+ log.error(OOM_FIX_RECOMMENDATION_LOG_MSG + validationResult, failureCtx.error());
+ }
CorruptedDataStructureException corruptedDataStructureEx =
X.cause(failureCtx.error(), CorruptedDataStructureException.class);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 1b6f9029518..11739560d21 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -304,6 +304,8 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_CACHE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_DATA_REGIONS_OFFHEAP_SIZE;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_JVM_PID;
import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_MACS;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_OFFHEAP_SIZE;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM;
import static org.apache.ignite.internal.util.GridUnsafe.objectFieldOffset;
import static org.apache.ignite.internal.util.GridUnsafe.putObjectVolatile;
import static org.apache.ignite.internal.util.GridUnsafe.staticFieldBase;
@@ -12510,4 +12512,52 @@ public abstract class IgniteUtils {
public static boolean isTxAwareQueriesEnabled(GridKernalContext kctx) {
return kctx.config().getTransactionConfiguration().isTxAwareQueriesEnabled();
}
+
+ /**
+ * @param ctx {@link GridKernalContext}.
+ * @return A warning message indicating excessive RAM usage or {@code null} if the RAM usage is within acceptable
+ * limits.
+ */
+ @SuppressWarnings("ConstantConditions")
+ public static String validateRamUsage(GridKernalContext ctx) {
+ long ram = ctx.discovery().localNode().attribute(ATTR_PHY_RAM);
+
+ if (ram != -1) {
+ String macs = ctx.discovery().localNode().attribute(ATTR_MACS);
+
+ long totalHeap = 0;
+ long totalOffheap = 0;
+
+ for (ClusterNode node : ctx.discovery().allNodes()) {
+ if (macs.equals(node.attribute(ATTR_MACS))) {
+ long heap = node.metrics().getHeapMemoryMaximum();
+ Long offheap = node.<Long>attribute(ATTR_OFFHEAP_SIZE);
+
+ if (heap != -1)
+ totalHeap += heap;
+
+ if (offheap != null)
+ totalOffheap += offheap;
+ }
+ }
+
+ long total = totalHeap + totalOffheap;
+
+ if (total < 0)
+ total = Long.MAX_VALUE;
+
+ // 4GB or 20% of available memory is expected to be used by OS and user applications
+ long safeToUse = ram - Math.max(4L << 30, (long)(ram * 0.2));
+
+ if (total > safeToUse) {
+ return "The total amount of RAM configured for nodes running on the local host exceeds " +
+ "the recommended maximum value. This may lead to significant slowdown due to swapping, or even " +
+ "JVM/Ignite crash with OutOfMemoryError (please decrease JVM heap size, data region size or " +
+ "checkpoint buffer size) [configured=" + (total >> 20) + "MB, available=" + (ram >> 20) +
+ "MB, recommended=" + (safeToUse >> 20) + "MB]";
+ }
+ }
+
+ return null;
+ }
}
diff --git a/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java
index 3c4d0965448..ddef04caee2 100644
--- a/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/failure/OomFailureHandlerTest.java
@@ -24,6 +24,8 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
@@ -35,13 +37,26 @@ import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.services.Service;
import org.apache.ignite.services.ServiceContext;
-import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.ListeningTestLogger;
+import org.apache.ignite.testframework.LogListener;
import org.junit.Test;
+import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_PHY_RAM;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
+
/**
* Out of memory error failure handler test.
*/
public class OomFailureHandlerTest extends AbstractFailureHandlerTest {
+ /** Oom failure logger message. */
+ private static final String OOM_ON_RAM_FAILURE_LOG_MSG = "The total amount of RAM configured for nodes running on " +
+ "the local host exceeds the recommended maximum value. This may lead to significant slowdown due to swapping, " +
+ "or even JVM/Ignite crash with OutOfMemoryError (please decrease JVM heap size, data region size or checkpoint " +
+ "buffer size) ";
+
+ /** Listener log messages. */
+ private final ListeningTestLogger srvTestLog = new ListeningTestLogger(log);
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
@@ -54,6 +69,8 @@ public class OomFailureHandlerTest extends AbstractFailureHandlerTest {
cfg.setIncludeEventTypes(EventType.EVTS_ALL);
+ cfg.setGridLogger(srvTestLog);
+
return cfg;
}
@@ -72,19 +89,7 @@ public class OomFailureHandlerTest extends AbstractFailureHandlerTest {
IgniteEx ignite0 = startGrid(0);
IgniteEx ignite1 = startGrid(1);
- try {
- IgniteFuture<Boolean> res = ignite0.compute(ignite0.cluster().forNodeId(ignite1.cluster().localNode().id()))
- .callAsync(new IgniteCallable<Boolean>() {
- @Override public Boolean call() throws Exception {
- throw new OutOfMemoryError();
- }
- });
-
- res.get();
- }
- catch (Throwable ignore) {
- // Expected.
- }
+ failWithOutOfMemoryErrorViaCompute(ignite0, ignite1);
assertFailureState(ignite0, ignite1);
}
@@ -203,12 +208,61 @@ public class OomFailureHandlerTest extends AbstractFailureHandlerTest {
assertFailureState(ignite0, ignite1);
}
+ /**
+ * Test OOME with maxed RAM usage in configuration.
+ */
+ @Test
+ public void testConfigurationOomError() throws Exception {
+ LogListener logLsnr0 = LogListener.matches(OOM_ON_RAM_FAILURE_LOG_MSG)
+ .times(2)
+ .build();
+
+ srvTestLog.registerListener(logLsnr0);
+
+ IgniteEx ignite0 = startGrid(0);
+
+ long ram = ignite0.localNode().attribute(ATTR_PHY_RAM);
+
+ IgniteConfiguration cfg = getConfiguration(getTestIgniteInstanceName(1))
+ .setDataStorageConfiguration(new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setMaxSize((long)(0.9 * ram))));
+
+ IgniteEx ignite1 = startGrid(cfg);
+
+ failWithOutOfMemoryErrorViaCompute(ignite0, ignite1);
+
+ assertFailureState(ignite0, ignite1);
+
+ assertTrue(waitForCondition(logLsnr0::check, getTestTimeout()));
+ }
+
+ /**
+ * @param ignite0 instance that starts compute.
+ * @param ignite1 instance that handles compute and throws {@link OutOfMemoryError}.
+ */
+ private void failWithOutOfMemoryErrorViaCompute(IgniteEx ignite0, IgniteEx ignite1) {
+ try {
+ IgniteFuture<Boolean> res = ignite0.compute(ignite0.cluster().forNodeId(ignite1.cluster().localNode().id()))
+ .callAsync(new IgniteCallable<Boolean>() {
+ @Override public Boolean call() throws Exception {
+ throw new OutOfMemoryError();
+ }
+ });
+
+ res.get();
+ }
+ catch (Throwable ignore) {
+ // Expected.
+ }
+ }
+
/**
* @param igniteWork Working ignite instance.
* @param igniteFail Failed ignite instance.
*/
private static void assertFailureState(Ignite igniteWork, Ignite igniteFail) throws IgniteInterruptedCheckedException {
- assertTrue(GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ assertTrue(waitForCondition(new GridAbsPredicate() {
@Override public boolean apply() {
return dummyFailureHandler(igniteFail).failure();
}