You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2016/09/26 10:37:57 UTC
[3/6] brooklyn-server git commit: track and report on soft references
track and report on soft references
Project: http://git-wip-us.apache.org/repos/asf/brooklyn-server/repo
Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-server/commit/457c8d41
Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-server/tree/457c8d41
Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-server/diff/457c8d41
Branch: refs/heads/master
Commit: 457c8d41db37e095b709ac4c4bab96c1d33d0bd3
Parents: efd337d
Author: Alex Heneveld <al...@cloudsoftcorp.com>
Authored: Fri Sep 16 16:52:43 2016 +0100
Committer: Alex Heneveld <al...@cloudsoftcorp.com>
Committed: Fri Sep 16 16:53:20 2016 +0100
----------------------------------------------------------------------
.../mgmt/internal/BrooklynGarbageCollector.java | 25 +++++----
.../org/apache/brooklyn/util/guava/Maybe.java | 6 +++
.../util/javalang/MemoryUsageTracker.java | 53 +++++++++++++++++++-
3 files changed, 71 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/457c8d41/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java b/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
index 75e663a..a66821f 100644
--- a/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
+++ b/core/src/main/java/org/apache/brooklyn/core/mgmt/internal/BrooklynGarbageCollector.java
@@ -54,6 +54,7 @@ import org.apache.brooklyn.util.core.task.BasicExecutionManager;
import org.apache.brooklyn.util.core.task.ExecutionListener;
import org.apache.brooklyn.util.core.task.Tasks;
import org.apache.brooklyn.util.exceptions.Exceptions;
+import org.apache.brooklyn.util.guava.Maybe.SoftlyPresent;
import org.apache.brooklyn.util.javalang.MemoryUsageTracker;
import org.apache.brooklyn.util.text.Strings;
import org.apache.brooklyn.util.time.Duration;
@@ -103,14 +104,17 @@ public class BrooklynGarbageCollector {
+ "higher than this threshhold "
+ "(default >1 means never)", 2.0);
+ public static final ConfigKey<Boolean> TRACK_SOFT_MAYBE_USAGE = ConfigKeys.newBooleanConfigKey(
+ "brooklyn.gc.trackSoftMaybeUsage", "whether to track each maybe soft-reference and report usage", true);
+
/**
* should we check for tasks which are submitted by another but backgrounded, i.e. not a child of that task?
* default to yes, despite it can be some extra loops, to make sure we GC them promptly.
* @since 0.7.0 */
- // work offender is {@link DynamicSequentialTask} internal job tracker, but it is marked
+ // worst offender is {@link DynamicSequentialTask} internal job tracker, but it is marked
// transient so it is destroyed prompty; there may be others, however;
- // but OTOH it might be expensive to check for these all the time!
- // TODO probably we can set this false (remove this and related code),
+ // it doesn't seem to be expensive in practise, but if it becomes so (which seems plausible)
+ // then probably we can set this false (or even remove this and related code),
// and just rely on usual GC to pick up background tasks; the lifecycle of background task
// should normally be independent of the submitter. (DST was the exception, and marking
// transient there fixes the main problem, which is when the submitter is GC'd but the submitted is not,
@@ -166,6 +170,8 @@ public class BrooklynGarbageCollector {
this.storage = storage;
this.brooklynProperties = brooklynProperties;
+ if (brooklynProperties.getConfig(TRACK_SOFT_MAYBE_USAGE))
+ SoftlyPresent.getUsageTracker().enable();
executor = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
@Override public Thread newThread(Runnable r) {
@@ -206,10 +212,10 @@ public class BrooklynGarbageCollector {
double memUsage = 1.0 - 1.0*Runtime.getRuntime().freeMemory() / Runtime.getRuntime().totalMemory();
if (memUsage > brooklynProperties.getConfig(FORCE_CLEAR_SOFT_REFERENCES_ON_MEMORY_USAGE_LEVEL)) {
- LOG.info("Forcing brooklyn gc including soft references due to memory usage: "+getUsageString());
+ LOG.info("Forcing brooklyn gc including soft-reference cleansing due to memory usage: "+getUsageString());
MemoryUsageTracker.forceClearSoftReferences();
System.gc(); System.gc();
- LOG.info("Forced thorough brooklyn gc, usage now: "+getUsageString());
+ LOG.info("Forced cleansing brooklyn gc, usage now: "+getUsageString());
} else if (brooklynProperties.getConfig(DO_SYSTEM_GC)) {
// Can be very useful when tracking down OOMEs etc, where a lot of tasks are executing
// Empirically observed that (on OS X jvm at least) calling twice blocks - logs a significant
@@ -221,7 +227,6 @@ public class BrooklynGarbageCollector {
} catch (Throwable t) {
Exceptions.propagateIfFatal(t);
LOG.warn("Error during management-context GC: "+t, t);
- // previously we bailed on all errors, but I don't think we should do that -Alex
}
}
@@ -231,12 +236,10 @@ public class BrooklynGarbageCollector {
}
public static String makeBasicUsageString() {
+ int present = (int)Math.round(100.0*SoftlyPresent.getUsageTracker().getPercentagePresent());
return Strings.makeSizeString(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())+" / "+
- Strings.makeSizeString(Runtime.getRuntime().totalMemory()) + " memory" +
- // don't mention soft references; they're not very accurate
- // TODO would be nice to give an indication of soft usage or expiry rate;
- // a sampling of Maybe instances could be useful for that
-// " ("+Strings.makeSizeString(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed()) + " soft); "+
+ Strings.makeSizeString(Runtime.getRuntime().totalMemory()) + " memory; " +
+ (present>=0 ? present+"% soft-reference maybe retention (of "+SoftlyPresent.getUsageTracker().getTotalEntries()+"); " : "") +
Thread.activeCount()+" threads";
}
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/457c8d41/utils/common/src/main/java/org/apache/brooklyn/util/guava/Maybe.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/org/apache/brooklyn/util/guava/Maybe.java b/utils/common/src/main/java/org/apache/brooklyn/util/guava/Maybe.java
index 73c9761..722d4bb 100644
--- a/utils/common/src/main/java/org/apache/brooklyn/util/guava/Maybe.java
+++ b/utils/common/src/main/java/org/apache/brooklyn/util/guava/Maybe.java
@@ -30,6 +30,7 @@ import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import org.apache.brooklyn.util.javalang.JavaClassNames;
+import org.apache.brooklyn.util.javalang.MemoryUsageTracker.SoftUsageTracker;
import com.google.common.annotations.Beta;
import com.google.common.base.Function;
@@ -387,11 +388,13 @@ public abstract class Maybe<T> implements Serializable, Supplier<T> {
}
public static class SoftlyPresent<T> extends Maybe<T> {
+ private static final SoftUsageTracker TRACKER = new SoftUsageTracker();
private static final long serialVersionUID = 436799990500336015L;
private final SoftReference<T> value;
private Maybe<T> defaultValue;
protected SoftlyPresent(@Nonnull T value) {
this.value = new SoftReference<T>(value);
+ TRACKER.track(this, this.value);
}
@Override
public T get() {
@@ -423,6 +426,9 @@ public abstract class Maybe<T> implements Serializable, Supplier<T> {
this.defaultValue = defaultValue;
return this;
}
+
+ /** Returns the global usage tracker to determine how many references are soft */
+ public static SoftUsageTracker getUsageTracker() { return TRACKER; }
}
@Override
http://git-wip-us.apache.org/repos/asf/brooklyn-server/blob/457c8d41/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java b/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
index 7d09efd..acfdfaa 100644
--- a/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
+++ b/utils/common/src/main/java/org/apache/brooklyn/util/javalang/MemoryUsageTracker.java
@@ -26,6 +26,7 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.brooklyn.util.collections.MutableList;
import org.apache.brooklyn.util.text.Strings;
+import com.google.common.annotations.Beta;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.RemovalListener;
@@ -34,9 +35,21 @@ import com.google.common.cache.RemovalNotification;
/**
* Tracks the amount of memory consumed by the given objects in use.
* <p>
- * {@link WeakReference}s are used internally, so that shortly after a {@link #track(Object, long)}ed object is GC'd,
+ * In one tracker, {@link WeakReference}s are used internally, so that shortly after a {@link #track(Object, long)}ed object is GC'd,
* the {@link #getBytesUsed()} value decrements appropriately.
+ * <p>
+ * In another we let clients listen for {@link SoftReference} expiration again using weak references.
+ * <p>
+ * Both only work with clients who opt-in with calls to these methods/instances.
+ * <p>
+ * This also includes a technique for clearing soft references.
+ * If you have access to a console you can force it with:
+ *
+ * <code>
+ * org.apache.brooklyn.util.javalang.MemoryUsageTracker.forceClearSoftReferences()
+ * </code>
*/
+@Beta // made beta in 0.10.0 due to dodgy nature of it
public class MemoryUsageTracker {
/**
@@ -92,6 +105,7 @@ public class MemoryUsageTracker {
final long HEADROOM = 1000*1000;
long lastAmount = 0;
long nextAmount = 0;
+ long oldUsed = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
try {
List<byte[]> dd = MutableList.of();
while (true) {
@@ -106,8 +120,43 @@ public class MemoryUsageTracker {
lastAmount = nextAmount;
}
} catch (OutOfMemoryError e) { /* expected */ }
+ System.gc(); System.gc();
+ long newUsed = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
return "allocated " + Strings.makeSizeString((lastAmount+nextAmount)/2) +
(lastAmount<nextAmount ? " +- "+Strings.makeSizeString((nextAmount-lastAmount)/2) : "")
- +" really free memory in "+Strings.makeSizeString(maxChunk)+" chunks";
+ +" really free memory in "+Strings.makeSizeString(maxChunk)+" chunks; "
+ +"memory used from "+Strings.makeSizeString(oldUsed)+" -> "+Strings.makeSizeString(newUsed)+" / "+
+ Strings.makeSizeString(Runtime.getRuntime().totalMemory());
}
+
+ /** Tracking for soft usage through SoftlyPresent instances */
+ public static class SoftUsageTracker {
+ private Cache<Object,SoftReference<?>> cache = null;
+ public synchronized void enable() {
+ cache = CacheBuilder.newBuilder().weakKeys().build();
+ }
+ public synchronized void disable() {
+ cache = null;
+ }
+ public long getTotalEntries() {
+ return cache.size();
+ }
+ public synchronized double getPercentagePresent() {
+ if (cache==null) return -1;
+ int present=0, total=0;
+ for (SoftReference<?> sr: cache.asMap().values()) {
+ total++;
+ if (sr.get()!=null) present++;
+ }
+ if (total==0) return -1;
+ return 1.0*present / total;
+ }
+ public synchronized void track(Object key, SoftReference<?> ref) {
+ if (cache!=null) cache.put(key, ref);
+ }
+ public synchronized void untrack(Object key) {
+ if (cache!=null) cache.invalidate(key);
+ }
+ }
+
}