You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by kr...@apache.org on 2017/01/03 18:48:16 UTC
[05/50] lucene-solr:jira/solr-8593: LUCENE-7595: Improve
RAMUsageTester in test-framework to estimate memory usage of runtime classes
and work with Java 9 EA (b148+). Disable static field heap usage checker in
LuceneTestCase
LUCENE-7595: Improve RAMUsageTester in test-framework to estimate memory usage of runtime classes and work with Java 9 EA (b148+). Disable static field heap usage checker in LuceneTestCase
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/f29d2b56
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/f29d2b56
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/f29d2b56
Branch: refs/heads/jira/solr-8593
Commit: f29d2b5668296dfcdb8d650305449674faa29847
Parents: 262049f
Author: Uwe Schindler <us...@apache.org>
Authored: Thu Dec 29 01:56:23 2016 +0100
Committer: Uwe Schindler <us...@apache.org>
Committed: Thu Dec 29 01:56:23 2016 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 4 +
.../apache/lucene/search/TestLRUQueryCache.java | 3 +-
.../org/apache/lucene/util/LuceneTestCase.java | 90 ++++++++--------
.../org/apache/lucene/util/RamUsageTester.java | 102 ++++++++++++++++---
4 files changed, 139 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f29d2b56/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 255867d..7a118f1 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -208,6 +208,10 @@ Other
* LUCENE-7599: Simplify TestRandomChains using Java's built-in Predicate and
Function interfaces. (Ahmet Arslan via Adrien Grand)
+* LUCENE-7595: Improve RAMUsageTester in test-framework to estimate memory usage of
+ runtime classes and work with Java 9 EA (b148+). Disable static field heap usage
+ checker in LuceneTestCase. (Uwe Schindler, Dawid Weiss)
+
Build
* LUCENE-7387: fix defaultCodec in build.xml to account for the line ending (hossman)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f29d2b56/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
index 87382f9..9ebacf7 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestLRUQueryCache.java
@@ -265,6 +265,8 @@ public class TestLRUQueryCache extends LuceneTestCase {
// This test makes sure that by making the same assumptions as LRUQueryCache, RAMUsageTester
// computes the same memory usage.
public void testRamBytesUsedAgreesWithRamUsageTester() throws IOException {
+ assumeFalse("LUCENE-7595: RamUsageTester does not work exact in Java 9 (estimations for maps and lists)", Constants.JRE_IS_MINIMUM_JAVA9);
+
final LRUQueryCache queryCache = new LRUQueryCache(1 + random().nextInt(5), 1 + random().nextInt(10000), context -> random().nextBoolean());
// an accumulator that only sums up memory usage of referenced filters and doc id sets
final RamUsageTester.Accumulator acc = new RamUsageTester.Accumulator() {
@@ -379,7 +381,6 @@ public class TestLRUQueryCache extends LuceneTestCase {
// by the cache itself, not cache entries, and we want to make sure that
// memory usage is not grossly underestimated.
public void testRamBytesUsedConstantEntryOverhead() throws IOException {
- LuceneTestCase.assumeFalse("RamUsageTester does not fully work on Java 9", Constants.JRE_IS_MINIMUM_JAVA9);
final LRUQueryCache queryCache = new LRUQueryCache(1000000, 10000000, context -> true);
final RamUsageTester.Accumulator acc = new RamUsageTester.Accumulator() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f29d2b56/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
index 1848c4e..50139a0 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
@@ -598,51 +598,55 @@ public abstract class LuceneTestCase extends Assert {
* other.
*/
@ClassRule
- public static TestRule classRules = RuleChain
- .outerRule(new TestRuleIgnoreTestSuites())
- .around(ignoreAfterMaxFailures)
- .around(suiteFailureMarker = new TestRuleMarkFailure())
- .around(new TestRuleAssertionsRequired())
- .around(new TestRuleLimitSysouts(suiteFailureMarker))
- .around(tempFilesCleanupRule = new TestRuleTemporaryFilesCleanup(suiteFailureMarker))
- .around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) {
- @Override
- protected boolean accept(java.lang.reflect.Field field) {
- // Don't count known classes that consume memory once.
- if (STATIC_LEAK_IGNORED_TYPES.contains(field.getType().getName())) {
- return false;
+ public static TestRule classRules;
+ static {
+ RuleChain r = RuleChain.outerRule(new TestRuleIgnoreTestSuites())
+ .around(ignoreAfterMaxFailures)
+ .around(suiteFailureMarker = new TestRuleMarkFailure())
+ .around(new TestRuleAssertionsRequired())
+ .around(new TestRuleLimitSysouts(suiteFailureMarker))
+ .around(tempFilesCleanupRule = new TestRuleTemporaryFilesCleanup(suiteFailureMarker));
+ // TODO LUCENE-7595: Java 9 does not allow to look into runtime classes, so we have to fix the RAM usage checker!
+ if (!Constants.JRE_IS_MINIMUM_JAVA9) {
+ r = r.around(new StaticFieldsInvariantRule(STATIC_LEAK_THRESHOLD, true) {
+ @Override
+ protected boolean accept(java.lang.reflect.Field field) {
+ // Don't count known classes that consume memory once.
+ if (STATIC_LEAK_IGNORED_TYPES.contains(field.getType().getName())) {
+ return false;
+ }
+ // Don't count references from ourselves, we're top-level.
+ if (field.getDeclaringClass() == LuceneTestCase.class) {
+ return false;
+ }
+ return super.accept(field);
}
- // Don't count references from ourselves, we're top-level.
- if (field.getDeclaringClass() == LuceneTestCase.class) {
- return false;
+ });
+ }
+ classRules = r.around(new NoClassHooksShadowingRule())
+ .around(new NoInstanceHooksOverridesRule() {
+ @Override
+ protected boolean verify(Method key) {
+ String name = key.getName();
+ return !(name.equals("setUp") || name.equals("tearDown"));
}
- return super.accept(field);
- }
- })
- .around(new NoClassHooksShadowingRule())
- .around(new NoInstanceHooksOverridesRule() {
- @Override
- protected boolean verify(Method key) {
- String name = key.getName();
- return !(name.equals("setUp") || name.equals("tearDown"));
- }
- })
- .around(classNameRule = new TestRuleStoreClassName())
- .around(new TestRuleRestoreSystemProperties(
- // Enlist all properties to which we have write access (security manager);
- // these should be restored to previous state, no matter what the outcome of the test.
-
- // We reset the default locale and timezone; these properties change as a side-effect
- "user.language",
- "user.timezone",
-
- // TODO: these should, ideally, be moved to Solr's base class.
- "solr.directoryFactory",
- "solr.solr.home",
- "solr.data.dir"
- ))
- .around(classEnvRule = new TestRuleSetupAndRestoreClassEnv());
-
+ })
+ .around(classNameRule = new TestRuleStoreClassName())
+ .around(new TestRuleRestoreSystemProperties(
+ // Enlist all properties to which we have write access (security manager);
+ // these should be restored to previous state, no matter what the outcome of the test.
+
+ // We reset the default locale and timezone; these properties change as a side-effect
+ "user.language",
+ "user.timezone",
+
+ // TODO: these should, ideally, be moved to Solr's base class.
+ "solr.directoryFactory",
+ "solr.solr.home",
+ "solr.data.dir"
+ ))
+ .around(classEnvRule = new TestRuleSetupAndRestoreClassEnv());
+ }
// -----------------------------------------------------------------
// Test level rules.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/f29d2b56/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
index 9850526..daf81a9 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
@@ -16,9 +16,12 @@
*/
package org.apache.lucene.util;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
import java.lang.reflect.Array;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
+import java.nio.file.Path;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.AbstractList;
@@ -30,6 +33,10 @@ import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.ToLongFunction;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
/** Crawls object graph to collect RAM usage for testing */
public final class RamUsageTester {
@@ -40,9 +47,7 @@ public final class RamUsageTester {
/** Accumulate transitive references for the provided fields of the given
* object into <code>queue</code> and return the shallow size of this object. */
public long accumulateObject(Object o, long shallowSize, Map<Field, Object> fieldValues, Collection<Object> queue) {
- for (Object value : fieldValues.values()) {
- queue.add(value);
- }
+ queue.addAll(fieldValues.values());
return shallowSize;
}
@@ -130,10 +135,10 @@ public final class RamUsageTester {
@Override
public int size() {
return len;
- }
-
- };
- }
+ }
+
+ };
+ }
totalSize += accumulator.accumulateArray(ob, shallowSize, values, stack);
} else {
/*
@@ -145,13 +150,36 @@ public final class RamUsageTester {
if (cachedInfo == null) {
classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
}
-
- Map<Field, Object> fieldValues = new HashMap<>();
- for (Field f : cachedInfo.referenceFields) {
- fieldValues.put(f, f.get(ob));
+
+ boolean needsReflection = true;
+ if (Constants.JRE_IS_MINIMUM_JAVA9) {
+ // Java 9: Best guess for some known types, as we cannot precisely look into runtime classes:
+ final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
+ if (func != null) { // some simple type like String where the size is easy to get from public properties
+ totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize + func.applyAsLong(ob),
+ Collections.emptyMap(), stack);
+ needsReflection = false;
+ } else if (ob instanceof Iterable) {
+ final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
+ .collect(Collectors.toList());
+ totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+ needsReflection = false;
+ } else if (ob instanceof Map) {
+ final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
+ .flatMap(e -> Stream.of(e.getKey(), e.getValue()))
+ .collect(Collectors.toList());
+ totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+ totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+ needsReflection = false;
+ }
+ }
+ if (needsReflection) {
+ final Map<Field, Object> fieldValues = new HashMap<>();
+ for (Field f : cachedInfo.referenceFields) {
+ fieldValues.put(f, f.get(ob));
+ }
+ totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
}
-
- totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize, fieldValues, stack);
} catch (IllegalAccessException e) {
// this should never happen as we enabled setAccessible().
throw new RuntimeException("Reflective field access failed?", e);
@@ -167,7 +195,41 @@ public final class RamUsageTester {
return totalSize;
}
-
+ /**
+ * This map contains a function to calculate sizes of some "simple types" like String just from their public properties.
+ * This is needed for Java 9, which does not allow to look into runtime class fields.
+ */
+ @SuppressWarnings("serial")
+ private static final Map<Class<?>, ToLongFunction<Object>> SIMPLE_TYPES = Collections.unmodifiableMap(new IdentityHashMap<Class<?>, ToLongFunction<Object>>() {
+ { init(); }
+
+ @SuppressForbidden(reason = "We measure some forbidden classes")
+ private void init() {
+ // String types:
+ a(String.class, v -> charArraySize(v.length())); // may not be correct with Java 9's compact strings!
+ a(StringBuilder.class, v -> charArraySize(v.capacity()));
+ a(StringBuffer.class, v -> charArraySize(v.capacity()));
+ // Types with large buffers:
+ a(ByteArrayOutputStream.class, v -> byteArraySize(v.size()));
+ // For File and Path, we just take the length of String representation as approximation:
+ a(File.class, v -> charArraySize(v.toString().length()));
+ a(Path.class, v -> charArraySize(v.toString().length()));
+ }
+
+ @SuppressWarnings("unchecked")
+ private <T> void a(Class<T> clazz, ToLongFunction<T> func) {
+ put(clazz, (ToLongFunction<Object>) func);
+ }
+
+ private long charArraySize(int len) {
+ return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)Character.BYTES * len);
+ }
+
+ private long byteArraySize(int len) {
+ return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
+ }
+ });
+
/**
* Cached information about a given class.
*/
@@ -202,8 +264,16 @@ public final class RamUsageTester {
shallowInstanceSize = RamUsageEstimator.adjustForField(shallowInstanceSize, f);
if (!f.getType().isPrimitive()) {
- f.setAccessible(true);
- referenceFields.add(f);
+ try {
+ f.setAccessible(true);
+ referenceFields.add(f);
+ } catch (RuntimeException re) {
+ if ("java.lang.reflect.InaccessibleObjectException".equals(re.getClass().getName())) {
+ // LUCENE-7595: this is Java 9, which prevents access to fields in foreign modules
+ } else {
+ throw re;
+ }
+ }
}
}
}