You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ds...@apache.org on 2017/04/14 18:00:19 UTC
geode git commit: GEODE-2097: check offheap limit during recovery
Repository: geode
Updated Branches:
refs/heads/develop 0714c27c9 -> 5c5c94797
GEODE-2097: check offheap limit during recovery
During disk recovery of offheap data,
the offheap LRU eviction limit is now checked
instead of the heap limit.
A new integration test has been added that would
run out of offheap memory during recovery without
this fix.
This fix also includes some refactoring of the
resource manager memory monitors into a new common
interface named MemoryMonitor.
Project: http://git-wip-us.apache.org/repos/asf/geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/geode/commit/5c5c9479
Tree: http://git-wip-us.apache.org/repos/asf/geode/tree/5c5c9479
Diff: http://git-wip-us.apache.org/repos/asf/geode/diff/5c5c9479
Branch: refs/heads/develop
Commit: 5c5c947978e8683f39dd9cae24fda4f6cd87f3e8
Parents: 0714c27
Author: Darrel Schneider <ds...@pivotal.io>
Authored: Wed Dec 21 11:40:30 2016 -0800
Committer: Darrel Schneider <ds...@pivotal.io>
Committed: Fri Apr 14 10:59:49 2017 -0700
----------------------------------------------------------------------
.../internal/cache/AbstractLRURegionMap.java | 16 +--
.../geode/internal/cache/AbstractRegionMap.java | 4 +-
.../geode/internal/cache/LocalRegion.java | 2 +-
.../internal/cache/PlaceHolderDiskRegion.java | 2 +-
.../geode/internal/cache/ProxyRegionMap.java | 4 +-
.../cache/control/HeapMemoryMonitor.java | 14 +--
.../cache/control/InternalResourceManager.java | 8 ++
.../internal/cache/control/MemoryMonitor.java | 39 +++++++
.../cache/control/OffHeapMemoryMonitor.java | 14 +--
.../geode/internal/cache/lru/EnableLRU.java | 3 +
.../cache/lru/HeapLRUCapacityController.java | 30 ++---
.../cache/lru/LRUCapacityController.java | 6 +
.../internal/cache/lru/LRUMapCallbacks.java | 5 +-
.../cache/lru/MemLRUCapacityController.java | 6 +
.../internal/cache/lru/LRUClockJUnitTest.java | 6 +
.../OffHeapLRURecoveryRegressionTest.java | 115 +++++++++++++++++++
16 files changed, 222 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java
index 328ff35..4d5bb87 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractLRURegionMap.java
@@ -36,6 +36,7 @@ import org.apache.geode.internal.cache.lru.LRUStatistics;
import org.apache.geode.internal.cache.lru.MemLRUCapacityController;
import org.apache.geode.internal.cache.lru.NewLIFOClockHand;
import org.apache.geode.internal.cache.lru.NewLRUClockHand;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
import org.apache.geode.internal.cache.versions.RegionVersionVector;
import org.apache.geode.internal.cache.versions.VersionSource;
import org.apache.geode.internal.i18n.LocalizedStrings;
@@ -572,15 +573,8 @@ public abstract class AbstractLRURegionMap extends AbstractRegionMap {
private boolean mustEvict() {
LocalRegion owner = _getOwner();
InternalResourceManager resourceManager = owner.getCache().getResourceManager();
-
- final boolean monitorStateIsEviction;
- if (!owner.getAttributes().getOffHeap()) {
- monitorStateIsEviction = resourceManager.getHeapMonitor().getState().isEviction();
- } else {
- monitorStateIsEviction = resourceManager.getOffHeapMonitor().getState().isEviction();
- }
-
- return monitorStateIsEviction && this.sizeInVM() > 0;
+ boolean offheap = owner.getAttributes().getOffHeap();
+ return resourceManager.getMemoryMonitor(offheap).getState().isEviction() && this.sizeInVM() > 0;
}
public final int centralizedLruUpdateCallback() {
@@ -856,8 +850,8 @@ public abstract class AbstractLRURegionMap extends AbstractRegionMap {
}
@Override
- public final boolean lruLimitExceeded() {
- return _getCCHelper().mustEvict(_getLruList().stats(), null, 0);
+ public final boolean lruLimitExceeded(DiskRegionView drv) {
+ return _getCCHelper().lruLimitExceeded(_getLruList().stats(), drv);
}
@Override
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java
index eaababa..bc9fcdf 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/AbstractRegionMap.java
@@ -31,6 +31,7 @@ import org.apache.geode.internal.cache.FilterRoutingInfo.FilterInfo;
import org.apache.geode.internal.cache.ha.HAContainerWrapper;
import org.apache.geode.internal.cache.ha.HARegionQueue;
import org.apache.geode.internal.cache.lru.LRUEntry;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
import org.apache.geode.internal.cache.region.entry.RegionEntryFactoryBuilder;
import org.apache.geode.internal.cache.tier.sockets.CacheClientNotifier;
import org.apache.geode.internal.cache.tier.sockets.ClientProxyMembershipID;
@@ -478,7 +479,8 @@ public abstract class AbstractRegionMap implements RegionMap {
e.decRefCount(null, lr);
}
- public boolean lruLimitExceeded() {
+ @Override
+ public boolean lruLimitExceeded(DiskRegionView drv) {
return false;
}
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
index d0aacc2..297c90b 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/LocalRegion.java
@@ -11132,7 +11132,7 @@ public class LocalRegion extends AbstractRegion implements LoaderHelperFactory,
}
public boolean lruLimitExceeded() {
- return this.entries.lruLimitExceeded();
+ return this.entries.lruLimitExceeded(getDiskRegionView());
}
public DiskEntry getDiskEntry(Object key) {
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java b/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java
index db01162..9946f26 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/PlaceHolderDiskRegion.java
@@ -127,7 +127,7 @@ public class PlaceHolderDiskRegion extends AbstractDiskRegion implements DiskRec
}
public boolean lruLimitExceeded() {
- return getRecoveredEntryMap().lruLimitExceeded();
+ return getRecoveredEntryMap().lruLimitExceeded(this);
}
public DiskStoreID getDiskStoreID() {
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java b/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java
index 92c7b6f..8ed07f8 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/ProxyRegionMap.java
@@ -35,6 +35,7 @@ import org.apache.geode.internal.InternalStatisticsDisabledException;
import org.apache.geode.internal.cache.AbstractRegionMap.ARMLockTestHook;
import org.apache.geode.internal.cache.lru.LRUEntry;
import org.apache.geode.internal.cache.lru.NewLRUClockHand;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
import org.apache.geode.internal.cache.tier.sockets.ClientProxyMembershipID;
import org.apache.geode.internal.cache.versions.RegionVersionVector;
import org.apache.geode.internal.cache.versions.VersionHolder;
@@ -366,7 +367,8 @@ final class ProxyRegionMap implements RegionMap {
// nothing needed
}
- public final boolean lruLimitExceeded() {
+ @Override
+ public final boolean lruLimitExceeded(DiskRegionView drv) {
return false;
}
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/control/HeapMemoryMonitor.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/control/HeapMemoryMonitor.java b/geode-core/src/main/java/org/apache/geode/internal/cache/control/HeapMemoryMonitor.java
index afc9a23..0de41bf 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/control/HeapMemoryMonitor.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/control/HeapMemoryMonitor.java
@@ -55,7 +55,7 @@ import java.util.concurrent.*;
*
* @since Geode 1.0
*/
-public class HeapMemoryMonitor implements NotificationListener, ResourceMonitor {
+public class HeapMemoryMonitor implements NotificationListener, MemoryMonitor {
private static final Logger logger = LogService.getLogger();
// Allow for an unknown heap pool for VMs we may support in the future.
@@ -397,10 +397,7 @@ public class HeapMemoryMonitor implements NotificationListener, ResourceMonitor
}
}
- float getCriticalThreshold() {
- return this.thresholds.getCriticalThreshold();
- }
-
+ @Override
public boolean hasEvictionThreshold() {
return this.hasEvictionThreshold;
}
@@ -449,10 +446,6 @@ public class HeapMemoryMonitor implements NotificationListener, ResourceMonitor
}
}
- public float getEvictionThreshold() {
- return this.thresholds.getEvictionThreshold();
- }
-
/**
* Compare the number of bytes used (fetched from the JVM) to the thresholds. If necessary, change
* the state and send an event for the state change.
@@ -539,10 +532,12 @@ public class HeapMemoryMonitor implements NotificationListener, ResourceMonitor
eventToPopulate.getThresholds());
}
+ @Override
public MemoryState getState() {
return this.currentState;
}
+ @Override
public MemoryThresholds getThresholds() {
MemoryThresholds saveThresholds = this.thresholds;
@@ -634,6 +629,7 @@ public class HeapMemoryMonitor implements NotificationListener, ResourceMonitor
/**
* Returns the number of bytes of memory reported by the tenured pool as currently in use.
*/
+ @Override
public long getBytesUsed() {
return getTenuredMemoryPoolMXBean().getUsage().getUsed();
}
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/control/InternalResourceManager.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/control/InternalResourceManager.java b/geode-core/src/main/java/org/apache/geode/internal/cache/control/InternalResourceManager.java
index d16cd98..e98b6db 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/control/InternalResourceManager.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/control/InternalResourceManager.java
@@ -265,6 +265,14 @@ public class InternalResourceManager implements ResourceManager {
return (OffHeapMemoryMonitor) this.resourceMonitors.get(ResourceType.OFFHEAP_MEMORY);
}
+ public MemoryMonitor getMemoryMonitor(boolean offheap) {
+ if (offheap) {
+ return getOffHeapMonitor();
+ } else {
+ return getHeapMonitor();
+ }
+ }
+
/**
* Use threshold event processor to execute the event embedded in the runnable.
*
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/control/MemoryMonitor.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/control/MemoryMonitor.java b/geode-core/src/main/java/org/apache/geode/internal/cache/control/MemoryMonitor.java
new file mode 100644
index 0000000..45d60f6
--- /dev/null
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/control/MemoryMonitor.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.internal.cache.control;
+
+import org.apache.geode.internal.cache.control.MemoryThresholds.MemoryState;
+
+/**
+ * A resource monitor that monitors memory.
+ */
+public interface MemoryMonitor extends ResourceMonitor {
+ public MemoryState getState();
+
+ public MemoryThresholds getThresholds();
+
+ public long getBytesUsed();
+
+ public boolean hasEvictionThreshold();
+
+ default public float getCriticalThreshold() {
+ return getThresholds().getCriticalThreshold();
+ }
+
+ default public float getEvictionThreshold() {
+ return getThresholds().getEvictionThreshold();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/control/OffHeapMemoryMonitor.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/control/OffHeapMemoryMonitor.java b/geode-core/src/main/java/org/apache/geode/internal/cache/control/OffHeapMemoryMonitor.java
index 4140367..c9fe6b6 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/control/OffHeapMemoryMonitor.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/control/OffHeapMemoryMonitor.java
@@ -37,7 +37,7 @@ import org.apache.geode.internal.offheap.MemoryUsageListener;
*
* @since Geode 1.0
*/
-public class OffHeapMemoryMonitor implements ResourceMonitor, MemoryUsageListener {
+public class OffHeapMemoryMonitor implements MemoryMonitor, MemoryUsageListener {
private static final Logger logger = LogService.getLogger();
private volatile MemoryThresholds thresholds = new MemoryThresholds(0);
private volatile MemoryEvent mostRecentEvent = new MemoryEvent(ResourceType.OFFHEAP_MEMORY,
@@ -230,10 +230,7 @@ public class OffHeapMemoryMonitor implements ResourceMonitor, MemoryUsageListene
}
}
- float getCriticalThreshold() {
- return this.thresholds.getCriticalThreshold();
- }
-
+ @Override
public boolean hasEvictionThreshold() {
return this.hasEvictionThreshold;
}
@@ -283,10 +280,6 @@ public class OffHeapMemoryMonitor implements ResourceMonitor, MemoryUsageListene
}
}
- public float getEvictionThreshold() {
- return this.thresholds.getEvictionThreshold();
- }
-
/**
* Compare the number of bytes used (fetched from the JVM) to the thresholds. If necessary, change
* the state and send an event for the state change.
@@ -415,10 +408,12 @@ public class OffHeapMemoryMonitor implements ResourceMonitor, MemoryUsageListene
eventToPopulate.getThresholds());
}
+ @Override
public MemoryState getState() {
return this.currentState;
}
+ @Override
public MemoryThresholds getThresholds() {
MemoryThresholds saveThresholds = this.thresholds;
@@ -429,6 +424,7 @@ public class OffHeapMemoryMonitor implements ResourceMonitor, MemoryUsageListene
/**
* Returns the number of bytes of memory reported by the memory allocator as currently in use.
*/
+ @Override
public long getBytesUsed() {
if (this.memoryAllocator == null) {
return 0;
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java
index 6aaf8cc..5beae60 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/EnableLRU.java
@@ -20,6 +20,7 @@ import org.apache.geode.StatisticsType;
import org.apache.geode.cache.EvictionAction;
import org.apache.geode.cache.EvictionAlgorithm;
import org.apache.geode.cache.Region;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
/**
* Marker interface to eviction controller that determines if LRU list maintainance is required.
@@ -108,5 +109,7 @@ public interface EnableLRU {
*/
public void afterEviction();
+ public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv);
+
}
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java
index 5e86ce8..cc2cd7e 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/HeapLRUCapacityController.java
@@ -26,6 +26,7 @@ import org.apache.geode.distributed.internal.DistributionConfig;
import org.apache.geode.internal.statistics.StatisticsTypeFactoryImpl;
import org.apache.geode.internal.cache.*;
import org.apache.geode.internal.cache.control.InternalResourceManager;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
import org.apache.geode.internal.i18n.LocalizedStrings;
import java.util.Properties;
@@ -267,31 +268,24 @@ public class HeapLRUCapacityController extends LRUAlgorithm {
* greater than the overflow threshold, then we evict the LRU entry.
*/
public boolean mustEvict(LRUStatistics stats, Region region, int delta) {
- final GemFireCacheImpl cache;
- if (region != null) {
- cache = (GemFireCacheImpl) region.getRegionService();
- } else {
- cache = GemFireCacheImpl.getInstance();
- }
+ final GemFireCacheImpl cache = (GemFireCacheImpl) region.getRegionService();
InternalResourceManager resourceManager = cache.getResourceManager();
-
- if (region == null) {
- return resourceManager.getHeapMonitor().getState().isEviction();
- }
-
- final boolean monitorStateIsEviction;
- if (!((AbstractRegion) region).getOffHeap()) {
- monitorStateIsEviction = resourceManager.getHeapMonitor().getState().isEviction();
- } else {
- monitorStateIsEviction = resourceManager.getOffHeapMonitor().getState().isEviction();
- }
-
+ boolean offheap = region.getAttributes().getOffHeap();
+ final boolean monitorStateIsEviction =
+ resourceManager.getMemoryMonitor(offheap).getState().isEviction();
if (region instanceof BucketRegion) {
return monitorStateIsEviction && ((BucketRegion) region).getSizeForEviction() > 0;
}
return monitorStateIsEviction && ((LocalRegion) region).getRegionMap().sizeInVM() > 0;
}
+
+ @Override
+ public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv) {
+ InternalResourceManager resourceManager =
+ drv.getDiskStore().getCache().getResourceManager();
+ return resourceManager.getMemoryMonitor(drv.getOffHeap()).getState().isEviction();
+ }
};
}
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java
index 3596a07..98e007c 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUCapacityController.java
@@ -18,6 +18,7 @@ import org.apache.geode.*;
import org.apache.geode.cache.*;
import org.apache.geode.internal.statistics.StatisticsTypeFactoryImpl;
import org.apache.geode.internal.cache.*;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
import org.apache.geode.internal.i18n.LocalizedStrings;
import java.util.*;
@@ -287,6 +288,11 @@ public final class LRUCapacityController extends LRUAlgorithm implements Declara
public boolean mustEvict(LRUStatistics stats, Region region, int delta) {
return stats.getCounter() + delta > stats.getLimit();
}
+
+ @Override
+ public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv) {
+ return lruStatistics.getCounter() > lruStatistics.getLimit();
+ }
};
}
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java
index 27a4ec0..0b1cc6f 100755
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/LRUMapCallbacks.java
@@ -17,6 +17,7 @@ package org.apache.geode.internal.cache.lru;
// import org.apache.geode.cache.Region;
import org.apache.geode.internal.cache.RegionEntry;
// import org.apache.geode.internal.cache.LocalRegion;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
/**
* The lru action on the map for evicting items must be called while the current thread is free of
@@ -54,8 +55,10 @@ public interface LRUMapCallbacks {
/**
* Return true if the lru has exceeded its limit and needs to evict. Note that this method is
* currently used to prevent disk recovery from faulting in values once the limit is exceeded.
+ *
+ * @param drv the disk region whose limit is checked
*/
- public boolean lruLimitExceeded();
+ public boolean lruLimitExceeded(DiskRegionView drv);
public void lruCloseStats();
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java
----------------------------------------------------------------------
diff --git a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java
index 2c2e8ec..1694f1b 100644
--- a/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java
+++ b/geode-core/src/main/java/org/apache/geode/internal/cache/lru/MemLRUCapacityController.java
@@ -31,6 +31,7 @@ import org.apache.geode.internal.statistics.StatisticsTypeFactoryImpl;
import org.apache.geode.internal.cache.AbstractLRURegionMap.CDValueWrapper;
import org.apache.geode.internal.cache.CachedDeserializableFactory;
import org.apache.geode.internal.cache.Token;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
import org.apache.geode.internal.i18n.LocalizedStrings;
@@ -421,6 +422,11 @@ public final class MemLRUCapacityController extends LRUAlgorithm implements Decl
public boolean mustEvict(LRUStatistics stats, Region region, int delta) {
return stats.getCounter() + delta > stats.getLimit();
}
+
+ @Override
+ public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv) {
+ return lruStatistics.getCounter() > lruStatistics.getLimit();
+ }
};
}
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java b/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java
index 8095d5a..39759f4 100755
--- a/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java
+++ b/geode-core/src/test/java/org/apache/geode/internal/cache/lru/LRUClockJUnitTest.java
@@ -40,6 +40,7 @@ import org.apache.geode.distributed.DistributedSystem;
import org.apache.geode.internal.statistics.StatisticsTypeFactoryImpl;
import org.apache.geode.internal.cache.InternalRegionArguments;
import org.apache.geode.internal.cache.PlaceHolderDiskRegion;
+import org.apache.geode.internal.cache.persistence.DiskRegionView;
import org.apache.geode.test.junit.categories.IntegrationTest;
/**
@@ -483,6 +484,11 @@ public class LRUClockJUnitTest {
stats.setLimit(limit());
return stats;
}
+
+ @Override
+ public boolean lruLimitExceeded(LRUStatistics lruStatistics, DiskRegionView drv) {
+ throw new UnsupportedOperationException("Not implemented");
+ }
}
/** overridden in SharedLRUClockTest to test SharedLRUClockHand */
http://git-wip-us.apache.org/repos/asf/geode/blob/5c5c9479/geode-core/src/test/java/org/apache/geode/internal/offheap/OffHeapLRURecoveryRegressionTest.java
----------------------------------------------------------------------
diff --git a/geode-core/src/test/java/org/apache/geode/internal/offheap/OffHeapLRURecoveryRegressionTest.java b/geode-core/src/test/java/org/apache/geode/internal/offheap/OffHeapLRURecoveryRegressionTest.java
new file mode 100644
index 0000000..489c62b
--- /dev/null
+++ b/geode-core/src/test/java/org/apache/geode/internal/offheap/OffHeapLRURecoveryRegressionTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.geode.internal.offheap;
+
+import static org.apache.geode.distributed.ConfigurationProperties.LOCATORS;
+import static org.apache.geode.distributed.ConfigurationProperties.MCAST_PORT;
+import static org.junit.Assert.assertEquals;
+
+import java.util.Properties;
+
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.geode.cache.CacheFactory;
+import org.apache.geode.cache.DiskStore;
+import org.apache.geode.cache.DiskStoreFactory;
+import org.apache.geode.cache.Region;
+import org.apache.geode.cache.RegionFactory;
+import org.apache.geode.cache.RegionShortcut;
+import org.apache.geode.distributed.ConfigurationProperties;
+import org.apache.geode.internal.cache.GemFireCacheImpl;
+import org.apache.geode.test.junit.categories.IntegrationTest;
+
+/**
+ * Test to reproduce GEODE-2097.
+ */
+@Category(IntegrationTest.class)
+public class OffHeapLRURecoveryRegressionTest {
+
+ static final String DS_NAME = "OffHeapLRURecoveryRegressionTestDS";
+
+ /**
+ * Test populates an offheap heaplru persistent region that contains more data than can fit in
+ * offheap memory. It then recovers the region to demonstrate that recovering this data will not
+ * try to put everything into offheap but instead leave some of it on disk.
+ */
+ @Test
+ public void recoveringTooMuchDataDoesNotRunOutOfOffHeapMemory() {
+ final int ENTRY_COUNT = 40;
+ final int expectedObjectCount;
+ GemFireCacheImpl gfc = createCache();
+ try {
+ Region<Object, Object> r = createRegion(gfc);
+ byte[] v = new byte[1024 * 1024];
+ for (int i = 0; i < ENTRY_COUNT; i++) {
+ r.put(i, v);
+ }
+ // expect one more during recovery because of the way the LRU limit is
+ // enforced during recover.
+ expectedObjectCount = MemoryAllocatorImpl.getAllocator().getStats().getObjects() + 1;
+ } finally {
+ gfc.close();
+ }
+ System.setProperty("gemfire.disk.recoverValuesSync", "true");
+ System.setProperty("gemfire.disk.recoverLruValues", "true");
+ try {
+ gfc = createCache();
+ try {
+ Region<Object, Object> r = createRegion(gfc);
+ try {
+ assertEquals(ENTRY_COUNT, r.size());
+ assertEquals(expectedObjectCount,
+ MemoryAllocatorImpl.getAllocator().getStats().getObjects());
+ } finally {
+ r.destroyRegion();
+ DiskStore ds = gfc.findDiskStore(DS_NAME);
+ ds.destroy();
+ }
+ } finally {
+ gfc.close();
+ }
+ } finally {
+ System.clearProperty("gemfire.disk.recoverValuesSync");
+ System.clearProperty("gemfire.disk.recoverLruValues");
+ }
+ }
+
+ private GemFireCacheImpl createCache() {
+ Properties props = new Properties();
+ props.setProperty(LOCATORS, "");
+ props.setProperty(MCAST_PORT, "0");
+ props.setProperty(ConfigurationProperties.OFF_HEAP_MEMORY_SIZE, "20m");
+ GemFireCacheImpl result = (GemFireCacheImpl) new CacheFactory(props).create();
+ result.getResourceManager().setEvictionOffHeapPercentage(50.0f);
+ return result;
+ }
+
+ private Region<Object, Object> createRegion(GemFireCacheImpl gfc) {
+ DiskStoreFactory dsf = gfc.createDiskStoreFactory();
+ dsf.create(DS_NAME);
+ RegionFactory<Object, Object> rf =
+ gfc.createRegionFactory(RegionShortcut.LOCAL_PERSISTENT_OVERFLOW);
+ rf.setOffHeap(true);
+ rf.setDiskStoreName(DS_NAME);
+ Region<Object, Object> r = rf.create("OffHeapLRURecoveryRegressionTestRegion");
+ return r;
+ }
+
+ private void closeCache(GemFireCacheImpl gfc) {
+ gfc.close();
+ MemoryAllocatorImpl.freeOffHeapMemory();
+ }
+}