You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2016/09/26 10:37:58 UTC
[4/6] brooklyn-server git commit: control over clearing soft
references (default disabled)
control over clearing soft references (default disabled)
and tidier memory usage reporting
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/efd337d9
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/efd337d9
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/efd337d9
Branch: refs/heads/master
Commit: efd337d92d340d8f98e516e9f466be3c89586541
Parents: efb0742
Author: Alex Heneveld <al...@cloudsoftcorp.com>
Authored: Fri Sep 16 14:27:26 2016 +0100
Committer: Alex Heneveld <al...@cloudsoftcorp.com>
Committed: Fri Sep 16 16:53:20 2016 +0100
----------------------------------------------------------------------
.../mgmt/internal/BrooklynGarbageCollector.java | 34 +++++++++++----
.../util/javalang/MemoryUsageTracker.java | 45 +++++++++++++++++++-
.../util/javalang/MemoryUsageTrackerTest.java | 34 ++++++++++++++-
3 files changed, 102 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/efd337d9/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
index c8ef0e6..75e663a 100644
--- a/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
+++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
@@ -60,8 +60,8 @@ import org.apache.brooklyn.util.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Objects;
import com.google.common.annotations.Beta;
+import com.google.common.base.Objects;
import com.google.common.collect.Iterables;
/**
@@ -92,8 +92,17 @@ public class BrooklynGarbageCollector {
Duration.minutes(1));
public static final ConfigKey<Boolean> DO_SYSTEM_GC = ConfigKeys.newBooleanConfigKey(
- "brooklyn.gc.doSystemGc", "whether to periodically call System.gc()", false);
-
+ "brooklyn.gc.doSystemGc", "whether to periodically call System.gc()", false);
+
+ public static final ConfigKey<Double> FORCE_CLEAR_SOFT_REFERENCES_ON_MEMORY_USAGE_LEVEL =
+ ConfigKeys.newDoubleConfigKey("brooklyn.gc.clearSoftReferencesOnMemoryUsageLevel",
+ "force clearance of soft references (by generating a deliberate OOME) "
+ + "if memory usage gets higher than this percentage of available memory; "
+ + "Brooklyn will use up to the max, or this percentage, with soft references,"
+ + "so if using any high-memory-usage alerts they should be pegged quite a bit"
+ + "higher than this threshhold "
+ + "(default >1 means never)", 2.0);
+
/**
* should we check for tasks which are submitted by another but backgrounded, i.e. not a child of that task?
* default to yes, despite it can be some extra loops, to make sure we GC them promptly.
@@ -142,6 +151,7 @@ public class BrooklynGarbageCollector {
};
private final BasicExecutionManager executionManager;
+ @SuppressWarnings("unused") // TODO remove BrooklynStorage altogether?
private final BrooklynStorage storage;
private final BrooklynProperties brooklynProperties;
private final ScheduledExecutorService executor;
@@ -149,7 +159,6 @@ public class BrooklynGarbageCollector {
private Map<Entity,Task<?>> unmanagedEntitiesNeedingGc = new LinkedHashMap<Entity, Task<?>>();
private Duration gcPeriod;
- private final boolean doSystemGc;
private volatile boolean running = true;
public BrooklynGarbageCollector(BrooklynProperties brooklynProperties, BasicExecutionManager executionManager, BrooklynStorage storage) {
@@ -157,7 +166,6 @@ public class BrooklynGarbageCollector {
this.storage = storage;
this.brooklynProperties = brooklynProperties;
- doSystemGc = brooklynProperties.getConfig(DO_SYSTEM_GC);
executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
@Override public Thread newThread(Runnable r) {
@@ -196,7 +204,13 @@ public class BrooklynGarbageCollector {
gcTasks();
logUsage("brooklyn gc (after)");
- if (doSystemGc) {
+ double memUsage = 1.0 - 1.0*Runtime.getRuntime().freeMemory() / Runtime.getRuntime().totalMemory();
+ if (memUsage > brooklynProperties.getConfig(FORCE_CLEAR_SOFT_REFERENCES_ON_MEMORY_USAGE_LEVEL)) {
+ LOG.info("Forcing brooklyn gc including soft references due to memory usage: "+getUsageString());
+ MemoryUsageTracker.forceClearSoftReferences();
+ System.gc(); System.gc();
+ LOG.info("Forced thorough brooklyn gc, usage now: "+getUsageString());
+ } else if (brooklynProperties.getConfig(DO_SYSTEM_GC)) {
// Can be very useful when tracking down OOMEs etc, where a lot of tasks are executing
// Empirically observed that (on OS X jvm at least) calling twice blocks - logs a significant
// amount of memory having been released, as though a full-gc had been run. But this is highly
@@ -219,13 +233,17 @@ public class BrooklynGarbageCollector {
public static String makeBasicUsageString() {
return Strings.makeSizeString(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())+" / "+
Strings.makeSizeString(Runtime.getRuntime().totalMemory()) + " memory" +
- " ("+Strings.makeSizeString(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed()) + " soft); "+
+ // don't mention soft references; they're not very accurate
+ // TODO would be nice to give an indication of soft usage or expiry rate;
+ // a sampling of Maybe instances could be useful for that
+// " ("+Strings.makeSizeString(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed()) + " soft); "+
Thread.activeCount()+" threads";
}
public String getUsageString() {
return makeBasicUsageString()+"; "+
- "storage: " + storage.getStorageMetrics() + "; " +
+ // ignore storage
+// "storage: " + storage.getStorageMetrics() + "; " +
"tasks: " +
executionManager.getNumActiveTasks()+" active, "+
executionManager.getNumIncompleteTasks()+" unfinished; "+
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/efd337d9/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java b/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
index e3897cc..7d09efd 100644
--- a/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
+++ b/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
@@ -20,8 +20,12 @@ package org.apache.brooklyn.util.javalang;
import java.lang.ref.SoftReference;
import java.lang.ref.WeakReference;
+import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.brooklyn.util.collections.MutableList;
+import org.apache.brooklyn.util.text.Strings;
+
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
@@ -44,7 +48,9 @@ public class MemoryUsageTracker {
* the amount of used memory which is reclaimable by collecting soft references.
* <p>
* This is particularly handy for tracking {@link SoftReference}s, because otherwise you can quickly get to a state
- * where {@link Runtime#freeMemory()} looks very low.
+ * where {@link Runtime#freeMemory()} <i>looks</i> very low.
+ * <p>
+ * Also consider {@link #forceClearSoftReferences()} to get useful information.
**/
public static final MemoryUsageTracker SOFT_REFERENCES = new MemoryUsageTracker();
@@ -68,5 +74,40 @@ public class MemoryUsageTracker {
memoryTrackedReferences.cleanUp();
return bytesUsed.get();
}
-
+
+ /** forces all soft references to be cleared by trying to allocate an enormous chunk of memory,
+ * returns a description of what was done
+ * (tune with with {@link #forceClearSoftReferences(long, int)}
+ * for greater than 200M precision in the output message, if you really care about that) */
+ public static String forceClearSoftReferences() {
+ return forceClearSoftReferences(1000*1000, Integer.MAX_VALUE);
+ }
+ /** as {@link #forceClearSoftReferences()} but gives control over headroom and max chunk size.
+ * it tries to undershoot by headroom as it approaches maximum (and then overshoot)
+ * to minimize the chance we take exactly all the memory and starve another thread;
+ * and it uses the given max chunk size in cases where the caller wants more precision
+ * (the available memory will be fragmented so the smaller the chunks the more it can
+ * fill in, but at the expense of time and actual memory provisioned) */
+ public static String forceClearSoftReferences(long headroom, int maxChunk) {
+ final long HEADROOM = 1000*1000;
+ long lastAmount = 0;
+ long nextAmount = 0;
+ try {
+ List<byte[]> dd = MutableList.of();
+ while (true) {
+ int size = (int)Math.min(Runtime.getRuntime().freeMemory()-HEADROOM, maxChunk);
+ if (size<HEADROOM) {
+ // do this to minimize the chance another thread gets an OOME
+ // due to us leaving just a tiny amount of memory
+ size = (int) Math.min(size + 2*HEADROOM, maxChunk);
+ }
+ nextAmount += size;
+ dd.add(new byte[size]);
+ lastAmount = nextAmount;
+ }
+ } catch (OutOfMemoryError e) { /* expected */ }
+ return "allocated " + Strings.makeSizeString((lastAmount+nextAmount)/2) +
+ (lastAmount<nextAmount ? " +- "+Strings.makeSizeString((nextAmount-lastAmount)/2) : "")
+ +" really free memory in "+Strings.makeSizeString(maxChunk)+" chunks";
+ }
}
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/efd337d9/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java
----------------------------------------------------------------------
diff --git a/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java b/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java
index 80e4e7f..f995e25 100644
--- a/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java
+++ b/utils/common/src/test/java/org/apache/brooklyn/util/javalang/MemoryUsageTrackerTest.java
@@ -81,7 +81,39 @@ public class MemoryUsageTrackerTest {
}
return size;
}
-
+
+ @Test(groups="Integration")
+ public void testSoftUsageAndClearance() {
+ long totalMemory = Runtime.getRuntime().totalMemory();
+ long freeMemory = Runtime.getRuntime().freeMemory();
+
+ List<Maybe<?>> dump = MutableList.of();
+ Maybe<byte[]> first = Maybe.soft(new byte[1000*1000]);
+ dump.add(first);
+ for (int i=0; i<1000*1000; i++) {
+ totalMemory = Runtime.getRuntime().totalMemory();
+ freeMemory = Runtime.getRuntime().freeMemory();
+
+ dump.add(Maybe.soft(new byte[1000*1000]));
+ if (first.isAbsent()) break;
+ }
+ int cleared = 0;
+ for (Maybe<?> m: dump) { if (m.isAbsent()) cleared++; }
+ LOG.info("First soft reference cleared after "+dump.size()+" 1M blocks created; "+cleared+" of them cleared");
+
+ Assert.assertTrue(1.0*freeMemory/totalMemory < 0.10,
+ "Should have had less than 10% free memory before clearance, had "+Strings.makeSizeString(freeMemory)+" / "+Strings.makeSizeString(totalMemory));
+
+ LOG.info("Forcing memory eviction: "+
+ MemoryUsageTracker.forceClearSoftReferences(100*1000, 10*1000*1000));
+
+ System.gc(); System.gc();
+ totalMemory = Runtime.getRuntime().totalMemory();
+ freeMemory = Runtime.getRuntime().freeMemory();
+ Assert.assertTrue(1.0*freeMemory/totalMemory > 0.90,
+ "Should now have more than 90% free memory, had "+Strings.makeSizeString(freeMemory)+" / "+Strings.makeSizeString(totalMemory));
+ }
+
private static void assertLessThan(long lhs, long rhs) {
Assert.assertTrue(lhs<rhs, "Expected "+lhs+" < "+rhs);
}