You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2019/01/17 11:25:39 UTC
[lucene-solr] branch branch_7x updated: LUCENE-8642,
LUCENE-8641: correct RamUsageTester.sizeOf's handling of
ByteBuffers. Throw exceptions on denied reflection to catch problems early.
This affects tests only.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch branch_7x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_7x by this push:
new ffd8e88 LUCENE-8642, LUCENE-8641: correct RamUsageTester.sizeOf's handling of ByteBuffers. Throw exceptions on denied reflection to catch problems early. This affects tests only.
ffd8e88 is described below
commit ffd8e88774e0eefbcb41720d7a4713f3cf6ad90c
Author: Dawid Weiss <dw...@apache.org>
AuthorDate: Thu Jan 17 12:23:30 2019 +0100
LUCENE-8642, LUCENE-8641: correct RamUsageTester.sizeOf's handling of ByteBuffers. Throw exceptions on denied reflection to catch problems early. This affects tests only.
---
.../apache/lucene/search/TermInSetQueryTest.java | 1 -
.../org/apache/lucene/util/RamUsageTester.java | 46 ++++++++++++----------
2 files changed, 26 insertions(+), 21 deletions(-)
diff --git a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
index 10761cb..ee14c3b 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TermInSetQueryTest.java
@@ -173,7 +173,6 @@ public class TermInSetQueryTest extends LuceneTestCase {
QueryUtils.checkEqual(query1, query2);
}
- @AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8641")
public void testRamBytesUsed() {
List<BytesRef> terms = new ArrayList<>();
final int numTerms = 1000 + random().nextInt(1000);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
index 6437d8e..6a15007 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
@@ -31,6 +31,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.function.ToLongFunction;
@@ -146,34 +147,41 @@ public final class RamUsageTester {
* and accumulate this object's shallow size.
*/
try {
- ClassCache cachedInfo = classCache.get(obClazz);
- if (cachedInfo == null) {
- classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
- }
-
boolean needsReflection = true;
if (Constants.JRE_IS_MINIMUM_JAVA9 && obClazz.getName().startsWith("java.")) {
+ long alignedShallowInstanceSize = RamUsageEstimator.shallowSizeOf(ob);
+
// Java 9: Best guess for some known types, as we cannot precisely look into runtime classes:
final ToLongFunction<Object> func = SIMPLE_TYPES.get(obClazz);
if (func != null) { // some simple type like String where the size is easy to get from public properties
- totalSize += accumulator.accumulateObject(ob, cachedInfo.alignedShallowInstanceSize + func.applyAsLong(ob),
+ totalSize += accumulator.accumulateObject(ob, alignedShallowInstanceSize + func.applyAsLong(ob),
Collections.emptyMap(), stack);
needsReflection = false;
- } else if (ob instanceof Iterable) {
- final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
- .collect(Collectors.toList());
- totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+ } else if (ob instanceof ByteBuffer) {
+ // Approximate ByteBuffers with their underlying storage (ignores field overhead).
+ totalSize += byteArraySize(((ByteBuffer) ob).capacity());
needsReflection = false;
} else if (ob instanceof Map) {
final List<Object> values = ((Map<?,?>) ob).entrySet().stream()
.flatMap(e -> Stream.of(e.getKey(), e.getValue()))
.collect(Collectors.toList());
- totalSize += accumulator.accumulateArray(ob, cachedInfo.alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+ totalSize += accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
totalSize += RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
needsReflection = false;
+ } else if (ob instanceof Iterable) {
+ final List<Object> values = StreamSupport.stream(((Iterable<?>) ob).spliterator(), false)
+ .collect(Collectors.toList());
+ totalSize += accumulator.accumulateArray(ob, alignedShallowInstanceSize + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER, values, stack);
+ needsReflection = false;
}
}
+
if (needsReflection) {
+ ClassCache cachedInfo = classCache.get(obClazz);
+ if (cachedInfo == null) {
+ classCache.put(obClazz, cachedInfo = createCacheEntry(obClazz));
+ }
+
final Map<Field, Object> fieldValues = new HashMap<>();
for (Field f : cachedInfo.referenceFields) {
fieldValues.put(f, f.get(ob));
@@ -224,10 +232,6 @@ public final class RamUsageTester {
private long charArraySize(int len) {
return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long)Character.BYTES * len);
}
-
- private long byteArraySize(int len) {
- return RamUsageEstimator.alignObjectSize((long)RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
- }
});
/**
@@ -268,11 +272,10 @@ public final class RamUsageTester {
f.setAccessible(true);
referenceFields.add(f);
} catch (RuntimeException re) {
- if ("java.lang.reflect.InaccessibleObjectException".equals(re.getClass().getName())) {
- // LUCENE-7595: this is Java 9, which prevents access to fields in foreign modules
- } else {
- throw re;
- }
+ throw new RuntimeException(String.format(Locale.ROOT,
+ "Can't access field %s of class %s for RAM estimation.",
+ f.toString(),
+ c.getName()), re);
}
}
}
@@ -286,4 +289,7 @@ public final class RamUsageTester {
});
}
+ private static long byteArraySize(int len) {
+ return RamUsageEstimator.alignObjectSize((long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + len);
+ }
}