You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by es...@apache.org on 2019/10/23 18:06:08 UTC
[geode] branch feature/GEODE-7341 updated: GEODE-7341: Provide a
way to avoid memory lock if over committed
This is an automated email from the ASF dual-hosted git repository.
eshu11 pushed a commit to branch feature/GEODE-7341
in repository https://gitbox.apache.org/repos/asf/geode.git
The following commit(s) were added to refs/heads/feature/GEODE-7341 by this push:
new e448752 GEODE-7341: Provide a way to avoid memory lock if over committed
e448752 is described below
commit e448752402b7cbf829e787197b121444b6a2bb50
Author: Eric Shu <es...@EricMacBookPro.local>
AuthorDate: Wed Oct 23 11:03:54 2019 -0700
GEODE-7341: Provide a way to avoid memory lock if over committed
---
.../internal/InternalDistributedSystem.java | 60 +++++++++-----
.../internal/InternalDistributedSystemTest.java | 95 ++++++++++++++++++++++
2 files changed, 134 insertions(+), 21 deletions(-)
diff --git a/geode-core/src/main/java/org/apache/geode/distributed/internal/InternalDistributedSystem.java b/geode-core/src/main/java/org/apache/geode/distributed/internal/InternalDistributedSystem.java
index c811243..99f8e56 100644
--- a/geode-core/src/main/java/org/apache/geode/distributed/internal/InternalDistributedSystem.java
+++ b/geode-core/src/main/java/org/apache/geode/distributed/internal/InternalDistributedSystem.java
@@ -135,11 +135,6 @@ import org.apache.geode.security.SecurityManager;
public class InternalDistributedSystem extends DistributedSystem
implements LogConfigSupplier {
- /**
- * True if the user is allowed lock when memory resources appear to be overcommitted.
- */
- private static final boolean ALLOW_MEMORY_LOCK_WHEN_OVERCOMMITTED =
- Boolean.getBoolean(GEMFIRE_PREFIX + "Cache.ALLOW_MEMORY_OVERCOMMIT");
private static final Logger logger = LogService.getLogger();
private static final String DISABLE_MANAGEMENT_PROPERTY =
@@ -172,6 +167,16 @@ public class InternalDistributedSystem extends DistributedSystem
private final StatisticsManager statisticsManager;
private final FunctionStatsManager functionStatsManager;
+ /**
+ * True if the user is allowed lock when memory resources appear to be overcommitted.
+ */
+ private final boolean allowMemoryLockWhenOvercommitted =
+ Boolean.getBoolean(GEMFIRE_PREFIX + "Cache.ALLOW_MEMORY_OVERCOMMIT");
+ /**
+ * True if memory lock is avoided when memory resources appear to be overcommitted.
+ */
+ private final boolean avoidMemoryLockWhenOvercommitted =
+ Boolean.getBoolean(GEMFIRE_PREFIX + "Cache.AVOID_MEMORY_LOCK_WHEN_OVERCOMMIT");
/**
* The distribution manager that is used to communicate with the distributed system.
@@ -724,22 +729,7 @@ public class InternalDistributedSystem extends DistributedSystem
// included the available memory calculation.
long avail = LinuxProcFsStatistics.getAvailableMemory(logger);
long size = offHeapMemorySize + Runtime.getRuntime().totalMemory();
- if (avail < size) {
- if (ALLOW_MEMORY_LOCK_WHEN_OVERCOMMITTED) {
- logger.warn(
- "System memory appears to be over committed by {} bytes. You may experience instability, performance issues, or terminated processes due to the Linux OOM killer.",
- size - avail);
- } else {
- throw new IllegalStateException(
- String.format(
- "Insufficient free memory (%s) when attempting to lock %s bytes. Either reduce the amount of heap or off-heap memory requested or free up additional system memory. You may also specify -Dgemfire.Cache.ALLOW_MEMORY_OVERCOMMIT=true on the command-line to override the constraint check.",
- avail, size));
- }
- }
-
- logger.info("Locking memory. This may take a while...");
- GemFireCacheImpl.lockMemory();
- logger.info("Finished locking memory.");
+ lockMemory(avail, size);
}
try {
@@ -817,6 +807,34 @@ public class InternalDistributedSystem extends DistributedSystem
reconnectAttemptCounter.set(0);
}
+ void lockMemory(long avail, long size) {
+ if (avail < size) {
+ if (avoidMemoryLockWhenOvercommitted) {
+ logger.warn(
+ "System memory appears to be over committed by {} bytes. Avoid memory lock.",
+ size - avail);
+ } else if (allowMemoryLockWhenOvercommitted) {
+ logger.warn(
+ "System memory appears to be over committed by {} bytes. You may experience instability, performance issues, or terminated processes due to the Linux OOM killer.",
+ size - avail);
+ lockMemory();
+ } else {
+ throw new IllegalStateException(
+ String.format(
+ "Insufficient free memory (%s) when attempting to lock %s bytes. Either reduce the amount of heap or off-heap memory requested or free up additional system memory. You may also specify -Dgemfire.Cache.ALLOW_MEMORY_OVERCOMMIT=true on the command-line to override the constraint check.",
+ avail, size));
+ }
+ } else {
+ lockMemory();
+ }
+ }
+
+ void lockMemory() {
+ logger.info("Locking memory. This may take a while...");
+ GemFireCacheImpl.lockMemory();
+ logger.info("Finished locking memory.");
+ }
+
private void startSampler() {
if (statsDisabled) {
return;
diff --git a/geode-core/src/test/java/org/apache/geode/distributed/internal/InternalDistributedSystemTest.java b/geode-core/src/test/java/org/apache/geode/distributed/internal/InternalDistributedSystemTest.java
new file mode 100644
index 0000000..b75fc53
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/distributed/internal/InternalDistributedSystemTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.distributed.internal;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.catchThrowable;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+
+import java.util.Properties;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.contrib.java.lang.system.RestoreSystemProperties;
+
+
+public class InternalDistributedSystemTest {
+
+ @Rule
+ public RestoreSystemProperties restoreSystemProperties = new RestoreSystemProperties();
+
+ @Test
+ public void lockMemoryAllowedIfAllowMemoryOverCommitIsSet() {
+ System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "Cache.ALLOW_MEMORY_OVERCOMMIT", "true");
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties()).build());
+ doNothing().when(system).lockMemory();
+
+ system.lockMemory(100, 200);
+
+ verify(system).lockMemory();
+ }
+
+ @Test
+ public void lockMemoryAvoidedIfAvoidMemoryLockWhenOverCommitIsSet() {
+ System.setProperty(
+ DistributionConfig.GEMFIRE_PREFIX + "Cache.AVOID_MEMORY_LOCK_WHEN_OVERCOMMIT", "true");
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties()).build());
+
+ system.lockMemory(100, 200);
+
+ verify(system, never()).lockMemory();
+ }
+
+ @Test
+ public void lockMemoryAvoidedIfAvoidAndAllowMemoryLockWhenOverCommitBothSet() {
+ System.setProperty(DistributionConfig.GEMFIRE_PREFIX + "Cache.ALLOW_MEMORY_OVERCOMMIT", "true");
+ System.setProperty(
+ DistributionConfig.GEMFIRE_PREFIX + "Cache.AVOID_MEMORY_LOCK_WHEN_OVERCOMMIT", "true");
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties()).build());
+
+ system.lockMemory(100, 200);
+
+ verify(system, never()).lockMemory();
+ }
+
+
+ @Test
+ public void lockMemoryThrowsIfMemoryOverCommit() {
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties()).build());
+
+ Throwable caughtException = catchThrowable(() -> system.lockMemory(100, 200));
+
+ assertThat(caughtException).isInstanceOf(IllegalStateException.class);
+ verify(system, never()).lockMemory();
+ }
+
+ @Test
+ public void locksMemoryIfMemoryNotOverCommit() {
+ InternalDistributedSystem system =
+ spy(new InternalDistributedSystem.Builder(new Properties()).build());
+ doNothing().when(system).lockMemory();
+
+ system.lockMemory(200, 100);
+
+ verify(system).lockMemory();
+ }
+}