You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2014/06/16 16:42:34 UTC
svn commit: r1602881 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/java/org/apache/lucene/util/
lucene/core/src/java/org/apache/lucene/util/packed/
lucene/core/src/test/org/ap...
Author: jpountz
Date: Mon Jun 16 14:42:34 2014
New Revision: 1602881
URL: http://svn.apache.org/r1602881
Log:
LUCENE-5765: Add tests to OrdinalMap.ramBytesUsed.
Added:
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
- copied unchanged from r1602880, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestOrdinalMap.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java?rev=1602881&r1=1602880&r2=1602881&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java Mon Jun 16 14:42:34 2014
@@ -112,9 +112,9 @@ class FrozenBufferedUpdates {
}
binaryDVUpdates = allBinaryUpdates.toArray(new BinaryDocValuesUpdate[allBinaryUpdates.size()]);
- bytesUsed = (int) terms.ramBytesUsed() + queries.length * BYTES_PER_DEL_QUERY
- + numericUpdatesSize + numericDVUpdates.length * RamUsageEstimator.NUM_BYTES_OBJECT_REF
- + binaryUpdatesSize + binaryDVUpdates.length * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ bytesUsed = (int) (terms.ramBytesUsed() + queries.length * BYTES_PER_DEL_QUERY
+ + numericUpdatesSize + RamUsageEstimator.shallowSizeOf(numericDVUpdates)
+ + binaryUpdatesSize + RamUsageEstimator.shallowSizeOf(binaryDVUpdates));
numTermDeletes = deletes.numTermDeletes.get();
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java?rev=1602881&r1=1602880&r2=1602881&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiDocValues.java Mon Jun 16 14:42:34 2014
@@ -25,6 +25,7 @@ import org.apache.lucene.index.MultiTerm
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.AppendingPackedLongBuffer;
import org.apache.lucene.util.packed.MonotonicAppendingLongBuffer;
import org.apache.lucene.util.packed.PackedInts;
@@ -377,6 +378,9 @@ public class MultiDocValues {
// TODO: use more efficient packed ints structures?
// TODO: pull this out? its pretty generic (maps between N ord()-enabled TermsEnums)
public static class OrdinalMap implements Accountable {
+
+ private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(OrdinalMap.class);
+
// cache key of whoever asked for this awful thing
final Object owner;
// globalOrd -> (globalOrd - segmentOrd) where segmentOrd is the the ordinal in the first segment that contains this term
@@ -473,7 +477,7 @@ public class MultiDocValues {
@Override
public long ramBytesUsed() {
- long size = globalOrdDeltas.ramBytesUsed() + firstSegments.ramBytesUsed();
+ long size = BASE_RAM_BYTES_USED + globalOrdDeltas.ramBytesUsed() + firstSegments.ramBytesUsed() + RamUsageEstimator.shallowSizeOf(ordDeltas);
for (int i = 0; i < ordDeltas.length; i++) {
size += ordDeltas[i].ramBytesUsed();
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java?rev=1602881&r1=1602880&r2=1602881&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java Mon Jun 16 14:42:34 2014
@@ -307,6 +307,12 @@ public final class RamUsageEstimator {
return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_DOUBLE * arr.length);
}
+ /** Returns the shallow size in bytes of the Object[] object. */
+ // Use this method instead of #shallowSizeOf(Object) to avoid costly reflection
+ public static long shallowSizeOf(Object[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_OBJECT_REF * arr.length);
+ }
+
/**
* Estimates a "shallow" memory usage of the given object. For arrays, this will be the
* memory taken by array storage (no subreferences will be followed). For objects, this
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java?rev=1602881&r1=1602880&r2=1602881&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java Mon Jun 16 14:42:34 2014
@@ -194,7 +194,7 @@ abstract class AbstractAppendingLongBuff
// TODO: this is called per-doc-per-norms/dv-field, can we optimize this?
long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed())
+ (pending != null ? RamUsageEstimator.sizeOf(pending) : 0L)
- + RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * values.length); // values
+ + RamUsageEstimator.shallowSizeOf(values);
return bytesUsed + valuesBytes;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java?rev=1602881&r1=1602880&r2=1602881&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/AbstractPagedMutable.java Mon Jun 16 14:42:34 2014
@@ -107,7 +107,7 @@ abstract class AbstractPagedMutable<T ex
/** Return the number of bytes used by this object. */
public long ramBytesUsed() {
long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed());
- bytesUsed += RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) RamUsageEstimator.NUM_BYTES_OBJECT_REF * subMutables.length);
+ bytesUsed += RamUsageEstimator.alignObjectSize(RamUsageEstimator.shallowSizeOf(subMutables));
for (PackedInts.Mutable gw : subMutables) {
bytesUsed += gw.ramBytesUsed();
}
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java?rev=1602881&r1=1602880&r2=1602881&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/RamUsageTester.java Mon Jun 16 14:42:34 2014
@@ -29,19 +29,44 @@ import java.util.NoSuchElementException;
/** Crawls object graph to collect RAM usage for testing */
public final class RamUsageTester {
- /**
+ /**
+ * A {@link Filter} that accepts all fields.
+ */
+ private static final Filter DEFAULT_FILTER = new Filter() {
+
+ @Override
+ public boolean accept(Field field) {
+ return true;
+ }
+
+ };
+
+ /** A filter that allows to decide on what to take into account when measuring RAM usage. */
+ public static interface Filter {
+
+ /** Whether the provided field should be taken into account when measuring RAM usage. */
+ boolean accept(Field field);
+
+ }
+
+ /**
* Estimates the RAM usage by the given object. It will
* walk the object tree and sum up all referenced objects.
- *
+ *
* <p><b>Resource Usage:</b> This method internally uses a set of
* every object seen during traversals so it does allocate memory
* (it isn't side-effect free). After the method exits, this memory
* should be GCed.</p>
*/
+ public static long sizeOf(Object obj, Filter filter) {
+ return measureObjectSize(obj, filter);
+ }
+
+ /** Same as calling <code>sizeOf(obj, DEFAULT_FILTER)</code>. */
public static long sizeOf(Object obj) {
- return measureObjectSize(obj);
+ return sizeOf(obj, DEFAULT_FILTER);
}
-
+
/**
* Return a human-readable size of a given object.
* @see #sizeOf(Object)
@@ -50,14 +75,14 @@ public final class RamUsageTester {
public static String humanSizeOf(Object object) {
return RamUsageEstimator.humanReadableUnits(sizeOf(object));
}
-
+
/*
- * Non-recursive version of object descend. This consumes more memory than recursive in-depth
+ * Non-recursive version of object descend. This consumes more memory than recursive in-depth
* traversal but prevents stack overflows on long chains of objects
* or complex graphs (a max. recursion depth on my machine was ~5000 objects linked in a chain
- * so not too much).
+ * so not too much).
*/
- private static long measureObjectSize(Object root) {
+ private static long measureObjectSize(Object root, Filter filter) {
// Objects seen so far.
final IdentityHashSet<Object> seen = new IdentityHashSet<>();
// Class cache with reference Field and precalculated shallow size.
@@ -113,10 +138,12 @@ public final class RamUsageTester {
}
for (Field f : cachedInfo.referenceFields) {
- // Fast path to eliminate redundancies.
- final Object o = f.get(ob);
- if (o != null && !seen.contains(o)) {
- stack.add(o);
+ if (filter.accept(f)) {
+ // Fast path to eliminate redundancies.
+ final Object o = f.get(ob);
+ if (o != null && !seen.contains(o)) {
+ stack.add(o);
+ }
}
}