You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by he...@apache.org on 2014/11/11 15:17:07 UTC
[03/11] incubator-brooklyn git commit: track soft references (esp
script output) and report reclaimable memory as part of brooklyn-gc reporting
track soft references (esp script output) and report reclaimable memory as part of brooklyn-gc reporting
Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/b6068e3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/b6068e3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/b6068e3f
Branch: refs/heads/master
Commit: b6068e3febd005eb279cb5d9c7537a0d2ae0f100
Parents: 396c600
Author: Alex Heneveld <al...@cloudsoftcorp.com>
Authored: Tue Nov 11 01:11:27 2014 +0000
Committer: Alex Heneveld <al...@cloudsoftcorp.com>
Committed: Tue Nov 11 13:54:54 2014 +0000
----------------------------------------------------------------------
.../brooklyn/entity/basic/BrooklynTaskTags.java | 7 +-
.../internal/BrooklynGarbageCollector.java | 4 +-
.../main/java/brooklyn/util/guava/Maybe.java | 15 ++--
.../util/javalang/MemoryUsageTracker.java | 72 ++++++++++++++++++++
.../util/javalang/MemoryUsageTrackerTest.java | 67 ++++++++++++++++++
5 files changed, 157 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b6068e3f/core/src/main/java/brooklyn/entity/basic/BrooklynTaskTags.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/entity/basic/BrooklynTaskTags.java b/core/src/main/java/brooklyn/entity/basic/BrooklynTaskTags.java
index 959a405..5c92b43 100644
--- a/core/src/main/java/brooklyn/entity/basic/BrooklynTaskTags.java
+++ b/core/src/main/java/brooklyn/entity/basic/BrooklynTaskTags.java
@@ -38,6 +38,7 @@ import brooklyn.management.ExecutionManager;
import brooklyn.management.Task;
import brooklyn.management.entitlement.EntitlementContext;
import brooklyn.util.guava.Maybe;
+import brooklyn.util.javalang.MemoryUsageTracker;
import brooklyn.util.stream.Streams;
import brooklyn.util.task.TaskTags;
import brooklyn.util.task.Tasks;
@@ -208,14 +209,16 @@ public class BrooklynTaskTags extends TaskTags {
/** not a stream, but inserted with the same mechanism */
public static final String STREAM_ENV = "env";
+ private static final Maybe<ByteArrayOutputStream> STREAM_GARBAGE_COLLECTED_MAYBE = Maybe.of(Streams.byteArrayOfString("<contents-garbage-collected>"));
+
/** creates a tag suitable for marking a stream available on a task */
public static WrappedStream tagForStream(String streamType, ByteArrayOutputStream stream) {
return new WrappedStream(streamType, stream);
}
/** creates a tag suitable for marking a stream available on a task, but which might be GC'd */
public static WrappedStream tagForStreamSoft(String streamType, ByteArrayOutputStream stream) {
- Maybe<ByteArrayOutputStream> weakStream = Maybe.softThen(stream,
- Maybe.of(Streams.byteArrayOfString("<contents-garbage-collected>")));
+ MemoryUsageTracker.SOFT_REFERENCES.track(stream, stream.size());
+ Maybe<ByteArrayOutputStream> weakStream = Maybe.softThen(stream, STREAM_GARBAGE_COLLECTED_MAYBE);
return new WrappedStream(streamType,
Suppliers.compose(Functions.toStringFunction(), weakStream),
Suppliers.compose(Streams.sizeFunction(), weakStream));
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b6068e3f/core/src/main/java/brooklyn/management/internal/BrooklynGarbageCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/management/internal/BrooklynGarbageCollector.java b/core/src/main/java/brooklyn/management/internal/BrooklynGarbageCollector.java
index c69e6b3..ed9b227 100644
--- a/core/src/main/java/brooklyn/management/internal/BrooklynGarbageCollector.java
+++ b/core/src/main/java/brooklyn/management/internal/BrooklynGarbageCollector.java
@@ -54,6 +54,7 @@ import brooklyn.util.collections.MutableList;
import brooklyn.util.collections.MutableMap;
import brooklyn.util.collections.MutableSet;
import brooklyn.util.exceptions.Exceptions;
+import brooklyn.util.javalang.MemoryUsageTracker;
import brooklyn.util.task.BasicExecutionManager;
import brooklyn.util.task.ExecutionListener;
import brooklyn.util.task.Tasks;
@@ -219,7 +220,8 @@ public class BrooklynGarbageCollector {
public String getUsageString() {
return
Strings.makeSizeString(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())+" / "+
- Strings.makeSizeString(Runtime.getRuntime().totalMemory()) + " memory; "+
+ Strings.makeSizeString(Runtime.getRuntime().totalMemory()) + " memory" +
+ " ("+Strings.makeSizeString(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed()) + " soft); "+
"storage: " + storage.getStorageMetrics() + "; " +
"tasks: " +
executionManager.getNumActiveTasks()+" active, "+
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b6068e3f/utils/common/src/main/java/brooklyn/util/guava/Maybe.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/brooklyn/util/guava/Maybe.java b/utils/common/src/main/java/brooklyn/util/guava/Maybe.java
index 86c82fc..8d18057 100644
--- a/utils/common/src/main/java/brooklyn/util/guava/Maybe.java
+++ b/utils/common/src/main/java/brooklyn/util/guava/Maybe.java
@@ -248,11 +248,16 @@ public abstract class Maybe<T> implements Serializable, Supplier<T> {
@Override
public T get() {
T result = value.get();
- if (result==null) {
- if (defaultValue==null) throw new IllegalStateException("Softly present item has been GC'd");
- return defaultValue.get();
- }
- return result;
+ if (result!=null) return result;
+ if (defaultValue==null) throw new IllegalStateException("Softly present item has been GC'd");
+ return defaultValue.get();
+ }
+ @Override
+ public T orNull() {
+ T result = value.get();
+ if (result!=null) return result;
+ if (defaultValue==null) return null;
+ return defaultValue.orNull();
}
@Override
public boolean isPresent() {
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b6068e3f/utils/common/src/main/java/brooklyn/util/javalang/MemoryUsageTracker.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/brooklyn/util/javalang/MemoryUsageTracker.java b/utils/common/src/main/java/brooklyn/util/javalang/MemoryUsageTracker.java
new file mode 100644
index 0000000..e8bb896
--- /dev/null
+++ b/utils/common/src/main/java/brooklyn/util/javalang/MemoryUsageTracker.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package brooklyn.util.javalang;
+
+import java.lang.ref.SoftReference;
+import java.lang.ref.WeakReference;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+
+/**
+ * Tracks the amount of memory consumed by the given objects in use.
+ * <p>
+ * {@link WeakReference}s are used internally, so that shortly after a {@link #track(Object, long)}ed object is GC'd,
+ * the {@link #getBytesUsed()} value decrements appropriately.
+ */
+public class MemoryUsageTracker {
+
+ /**
+ * Shared instance for use for tracking memory used by {@link SoftReference}.
+ * <p>
+ * Callers should only use this field to {@link #track(Object, long)} objects which have (or will soon have)
+ * given up their strong references, so that only soft or weak references remain.
+ * Provided size estimates are accurate, {@link #getBytesUsed()} will report
+ * the amount of used memory which is reclaimable by collecting soft references.
+ * <p>
+ * This is particularly handy for tracking {@link SoftReference}s, because otherwise you can quickly get to a state
+ * where {@link Runtime#freeMemory()} looks very low.
+ **/
+ public static final MemoryUsageTracker SOFT_REFERENCES = new MemoryUsageTracker();
+
+ AtomicLong bytesUsed = new AtomicLong(0);
+
+ Cache<Object, Long> memoryTrackedReferences = CacheBuilder.newBuilder()
+ .weakKeys()
+ .removalListener(new RemovalListener<Object,Long>() {
+ @Override
+ public void onRemoval(RemovalNotification<Object, Long> notification) {
+ bytesUsed.addAndGet(-notification.getValue());
+ }
+ }).build();
+
+ public void track(Object instance, long bytesUsedByInstance) {
+ bytesUsed.addAndGet(bytesUsedByInstance);
+ memoryTrackedReferences.put(instance, bytesUsedByInstance);
+ }
+
+ public long getBytesUsed() {
+ memoryTrackedReferences.cleanUp();
+ return bytesUsed.get();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/b6068e3f/utils/common/src/test/java/brooklyn/util/javalang/MemoryUsageTrackerTest.java
----------------------------------------------------------------------
diff --git a/utils/common/src/test/java/brooklyn/util/javalang/MemoryUsageTrackerTest.java b/utils/common/src/test/java/brooklyn/util/javalang/MemoryUsageTrackerTest.java
new file mode 100644
index 0000000..24f101b
--- /dev/null
+++ b/utils/common/src/test/java/brooklyn/util/javalang/MemoryUsageTrackerTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package brooklyn.util.javalang;
+
+import java.util.List;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import brooklyn.util.collections.MutableList;
+import brooklyn.util.guava.Maybe;
+import brooklyn.util.text.Strings;
+
+public class MemoryUsageTrackerTest {
+
+ @Test(groups="Integration")
+ public void testBigUsage() {
+ List<Maybe<byte[]>> references = MutableList.of();
+ long created = 0;
+ while (created < 2*Runtime.getRuntime().maxMemory()) {
+ byte d[] = new byte[1000000];
+ references.add(Maybe.soft(d));
+ MemoryUsageTracker.SOFT_REFERENCES.track(d, d.length);
+ created += d.length;
+
+ System.out.println("created "+Strings.makeSizeString(created) +
+ " ... in use: "+Strings.makeSizeString(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory())+" / " +
+ Strings.makeSizeString(Runtime.getRuntime().totalMemory()) +
+ " ... reclaimable: "+Strings.makeSizeString(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed()) +
+ " ... live refs: "+Strings.makeSizeString(sizeOfActiveReferences(references)));
+
+ assertLessThan(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed(), Runtime.getRuntime().maxMemory());
+ assertLessThan(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed(), Runtime.getRuntime().totalMemory());
+ assertLessThan(MemoryUsageTracker.SOFT_REFERENCES.getBytesUsed(), Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory());
+ }
+ }
+
+ private long sizeOfActiveReferences(List<Maybe<byte[]>> references) {
+ long size = 0;
+ for (Maybe<byte[]> ref: references) {
+ byte[] deref = ref.orNull();
+ if (deref!=null) size += deref.length;
+ }
+ return size;
+ }
+
+ private static void assertLessThan(long lhs, long rhs) {
+ Assert.assertTrue(lhs<rhs, "Expected "+lhs+" < "+rhs);
+ }
+
+}