You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/03/18 17:12:11 UTC
svn commit: r1302152 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/
lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/
lucene/core/src/java/org/apache/lucene/search/ lu...
Author: uschindler
Date: Sun Mar 18 16:12:11 2012
New Revision: 1302152
URL: http://svn.apache.org/viewvc?rev=1302152&view=rev
Log:
Merged revision(s) 1302133 from lucene/dev/trunk:
LUCENE-3867: Refactor RamUsageEstimator. CHANGES.txt will come separately...
Added:
lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java
- copied, changed from r1302133, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java
Removed:
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/MemoryModel.java
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/Constants.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java
lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java
Modified: lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Sun Mar 18 16:12:11 2012
@@ -53,6 +53,7 @@ import org.apache.lucene.search.Similari
import org.apache.lucene.store.RAMDirectory; // for javadocs
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Constants; // for javadocs
+import org.apache.lucene.util.RamUsageEstimator;
/**
* High-performance single-document main memory Apache Lucene fulltext search index.
@@ -471,38 +472,12 @@ public class MemoryIndex implements Seri
/**
* Returns a reasonable approximation of the main memory [bytes] consumed by
- * this instance. Useful for smart memory sensititive caches/pools. Assumes
- * fieldNames are interned, whereas tokenized terms are memory-overlaid.
- *
+ * this instance. Useful for smart memory sensititive caches/pools.
* @return the main memory consumption
*/
- public int getMemorySize() {
- // for example usage in a smart cache see nux.xom.pool.Pool
- int PTR = VM.PTR;
- int INT = VM.INT;
- int size = 0;
- size += VM.sizeOfObject(2*PTR + INT); // memory index
- if (sortedFields != null) size += VM.sizeOfObjectArray(sortedFields.length);
-
- size += VM.sizeOfHashMap(fields.size());
- for (Map.Entry<String, Info> entry : fields.entrySet()) { // for each Field Info
- Info info = entry.getValue();
- size += VM.sizeOfObject(2*INT + 3*PTR); // Info instance vars
- if (info.sortedTerms != null) size += VM.sizeOfObjectArray(info.sortedTerms.length);
-
- int len = info.terms.size();
- size += VM.sizeOfHashMap(len);
- Iterator<Map.Entry<String,ArrayIntList>> iter2 = info.terms.entrySet().iterator();
- while (--len >= 0) { // for each term
- Map.Entry<String,ArrayIntList> e = iter2.next();
- size += VM.sizeOfObject(PTR + 3*INT); // assumes substring() memory overlay
-// size += STR + 2 * ((String) e.getKey()).length();
- ArrayIntList positions = e.getValue();
- size += VM.sizeOfArrayIntList(positions.size());
- }
- }
- return size;
- }
+ public long getMemorySize() {
+ return RamUsageEstimator.sizeOf(this);
+ }
private int numPositions(ArrayIntList positions) {
return positions.size() / stride;
@@ -1195,61 +1170,4 @@ public class MemoryIndex implements Seri
if (DEBUG) System.err.println("MemoryIndexReader.doClose");
}
}
-
-
- ///////////////////////////////////////////////////////////////////////////////
- // Nested classes:
- ///////////////////////////////////////////////////////////////////////////////
- private static final class VM {
-
- public static final int PTR = Constants.JRE_IS_64BIT ? 8 : 4;
-
- public static final int INT = 4;
- private static final int LOG_PTR = (int) Math.round(log2(PTR));
-
- /**
- * Object header of any heap allocated Java object.
- * ptr to class, info for monitor, gc, hash, etc.
- */
- private static final int OBJECT_HEADER = 2*PTR;
-
- private VM() {} // not instantiable
-
- // assumes n > 0
- // 64 bit VM:
- // 0 --> 0*PTR
- // 1..8 --> 1*PTR
- // 9..16 --> 2*PTR
- private static int sizeOf(int n) {
- return (((n-1) >> LOG_PTR) + 1) << LOG_PTR;
- }
-
- public static int sizeOfObject(int n) {
- return sizeOf(OBJECT_HEADER + n);
- }
-
- public static int sizeOfObjectArray(int len) {
- return sizeOfObject(INT + PTR*len);
- }
-
- public static int sizeOfIntArray(int len) {
- return sizeOfObject(INT + INT*len);
- }
-
- public static int sizeOfHashMap(int len) {
- return sizeOfObject(4*PTR + 4*INT) + sizeOfObjectArray(len)
- + len * sizeOfObject(3*PTR + INT); // entries
- }
-
- public static int sizeOfArrayIntList(int len) {
- return sizeOfObject(PTR + INT) + sizeOfIntArray(len);
- }
-
- /** logarithm to the base 2. Example: log2(4) == 2, log2(8) == 3 */
- private static double log2(double value) {
- return Math.log(value) / Math.log(2);
- }
-
- }
-
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/spellchecker/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java Sun Mar 18 16:12:11 2012
@@ -132,13 +132,12 @@ public class LookupBenchmarkTest extends
*/
public void testStorageNeeds() throws Exception {
System.err.println("-- RAM consumption");
- final RamUsageEstimator rue = new RamUsageEstimator();
for (Class<? extends Lookup> cls : benchmarkClasses) {
Lookup lookup = buildLookup(cls, dictionaryInput);
System.err.println(
String.format(Locale.ENGLISH, "%-15s size[B]:%,13d",
lookup.getClass().getSimpleName(),
- rue.estimateRamUsage(lookup)));
+ RamUsageEstimator.sizeOf(lookup)));
}
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/search/FieldCache.java Sun Mar 18 16:12:11 2012
@@ -609,22 +609,16 @@ public interface FieldCache {
protected final void setEstimatedSize(String size) {
this.size = size;
}
- /**
- * @see #estimateSize(RamUsageEstimator)
- */
- public void estimateSize() {
- estimateSize(new RamUsageEstimator(false)); // doesn't check for interned
- }
+
/**
* Computes (and stores) the estimated size of the cache Value
* @see #getEstimatedSize
*/
- public void estimateSize(RamUsageEstimator ramCalc) {
- long size = ramCalc.estimateRamUsage(getValue());
- setEstimatedSize(RamUsageEstimator.humanReadableUnits
- (size, new DecimalFormat("0.#")));
-
+ public void estimateSize() {
+ long size = RamUsageEstimator.sizeOf(getValue());
+ setEstimatedSize(RamUsageEstimator.humanReadableUnits(size));
}
+
/**
* The most recently estimated size of the value, null unless
* estimateSize has been called.
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/Constants.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/Constants.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/Constants.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/Constants.java Sun Mar 18 16:12:11 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import java.lang.reflect.Field;
import org.apache.lucene.LucenePackage;
/**
@@ -57,22 +58,36 @@ public final class Constants {
public static final String OS_VERSION = System.getProperty("os.version");
public static final String JAVA_VENDOR = System.getProperty("java.vendor");
- public static final boolean JRE_IS_64BIT;
public static final boolean JRE_IS_MINIMUM_JAVA6;
public static final boolean JRE_IS_MINIMUM_JAVA7;
+
+ /** True iff running on a 64bit JVM */
+ public static final boolean JRE_IS_64BIT;
+
static {
- // NOTE: this logic may not be correct; if you know of a
- // more reliable approach please raise it on java-dev!
- final String x = System.getProperty("sun.arch.data.model");
- if (x != null) {
- JRE_IS_64BIT = x.indexOf("64") != -1;
- } else {
- if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) {
- JRE_IS_64BIT = true;
+ boolean is64Bit = false;
+ try {
+ final Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
+ final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ final Object unsafe = unsafeField.get(null);
+ final int addressSize = ((Number) unsafeClass.getMethod("addressSize")
+ .invoke(unsafe)).intValue();
+ //System.out.println("Address size: " + addressSize);
+ is64Bit = addressSize >= 8;
+ } catch (Exception e) {
+ final String x = System.getProperty("sun.arch.data.model");
+ if (x != null) {
+ is64Bit = x.indexOf("64") != -1;
} else {
- JRE_IS_64BIT = false;
+ if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) {
+ is64Bit = true;
+ } else {
+ is64Bit = false;
+ }
}
}
+ JRE_IS_64BIT = is64Bit;
// this method only exists in Java 6:
boolean v6 = true;
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java Sun Mar 18 16:12:11 2012
@@ -51,16 +51,17 @@ import org.apache.lucene.search.FieldCac
*/
public final class FieldCacheSanityChecker {
- private RamUsageEstimator ramCalc = null;
+ private boolean estimateRam;
+
public FieldCacheSanityChecker() {
/* NOOP */
}
+
/**
- * If set, will be used to estimate size for all CacheEntry objects
- * dealt with.
+ * If set, estimate size for all CacheEntry objects will be calculateed.
*/
- public void setRamUsageEstimator(RamUsageEstimator r) {
- ramCalc = r;
+ public void setRamUsageEstimator(boolean flag) {
+ estimateRam = flag;
}
@@ -79,8 +80,7 @@ public final class FieldCacheSanityCheck
*/
public static Insanity[] checkSanity(CacheEntry... cacheEntries) {
FieldCacheSanityChecker sanityChecker = new FieldCacheSanityChecker();
- // doesn't check for interned
- sanityChecker.setRamUsageEstimator(new RamUsageEstimator(false));
+ sanityChecker.setRamUsageEstimator(true);
return sanityChecker.check(cacheEntries);
}
@@ -96,9 +96,9 @@ public final class FieldCacheSanityCheck
if (null == cacheEntries || 0 == cacheEntries.length)
return new Insanity[0];
- if (null != ramCalc) {
+ if (estimateRam) {
for (int i = 0; i < cacheEntries.length; i++) {
- cacheEntries[i].estimateSize(ramCalc);
+ cacheEntries[i].estimateSize();
}
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java Sun Mar 18 16:12:11 2012
@@ -17,194 +17,519 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import java.lang.management.ManagementFactory;
import java.lang.reflect.*;
import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
import java.util.*;
/**
- * Estimates the size of a given Object using a given MemoryModel for primitive
- * size information.
+ * Estimates the size of Java objects using a simple memory model
+ * for primitive size information.
*
- * Resource Usage:
+ * <p>NOTE: Starting with Lucene 3.6, creating instances of this class
+ * is deprecated. If you still do this, please note, that instances of
+ * {@code RamUsageEstimator} are not thread-safe!
+ * It is also deprecated to enable checking of String intern-ness,
+ * the new static method no longer allow to do this. Interned strings
+ * will be counted as any other object and count for memory usage.
*
- * Internally uses a Map to temporally hold a reference to every
- * object seen.
- *
- * If checkInterned, all Strings checked will be interned, but those
- * that were not already interned will be released for GC when the
- * estimate is complete.
+ * <p>In Lucene 3.6, custom {@code MemoryModel}s were completely
+ * removed. The new implementation is now using Hotspot™ internals
+ * to get the correct scale factors and offsets for calculating
+ * memory usage.
*
* @lucene.internal
*/
public final class RamUsageEstimator {
+
+ // private RamUsageEstimator() {} // no instance in Lucene 4.0
+ public final static int NUM_BYTES_BOOLEAN = 1;
+ public final static int NUM_BYTES_BYTE = 1;
+ public final static int NUM_BYTES_CHAR = 2;
public final static int NUM_BYTES_SHORT = 2;
public final static int NUM_BYTES_INT = 4;
- public final static int NUM_BYTES_LONG = 8;
public final static int NUM_BYTES_FLOAT = 4;
+ public final static int NUM_BYTES_LONG = 8;
public final static int NUM_BYTES_DOUBLE = 8;
- public final static int NUM_BYTES_CHAR = 2;
- public final static int NUM_BYTES_OBJECT_HEADER = 8;
- public final static int NUM_BYTES_OBJECT_REF = Constants.JRE_IS_64BIT ? 8 : 4;
- public final static int NUM_BYTES_ARRAY_HEADER = NUM_BYTES_OBJECT_HEADER + NUM_BYTES_INT + NUM_BYTES_OBJECT_REF;
-
- private MemoryModel memoryModel;
- private final Map<Object,Object> seen;
+ public final static int NUM_BYTES_OBJECT_REF;
+
+ public final static int NUM_BYTES_OBJECT_HEADER;
+ public final static int NUM_BYTES_ARRAY_HEADER;
+
+ /**
+ * A constant specifying the object alignment boundary inside the JVM. Objects will
+ * always take a full multiple of this constant, possibly wasting some space.
+ */
+ public final static int NUM_BYTES_OBJECT_ALIGNMENT;
- private int refSize;
- private int arraySize;
- private int classSize;
+ /**
+ * Sizes of primitive classes.
+ */
+ private static final Map<Class<?>,Integer> primitiveSizes;
+ static {
+ primitiveSizes = new IdentityHashMap<Class<?>,Integer>();
+ primitiveSizes.put(boolean.class, Integer.valueOf(NUM_BYTES_BOOLEAN));
+ primitiveSizes.put(byte.class, Integer.valueOf(NUM_BYTES_BYTE));
+ primitiveSizes.put(char.class, Integer.valueOf(NUM_BYTES_CHAR));
+ primitiveSizes.put(short.class, Integer.valueOf(NUM_BYTES_SHORT));
+ primitiveSizes.put(int.class, Integer.valueOf(NUM_BYTES_INT));
+ primitiveSizes.put(float.class, Integer.valueOf(NUM_BYTES_FLOAT));
+ primitiveSizes.put(double.class, Integer.valueOf(NUM_BYTES_DOUBLE));
+ primitiveSizes.put(long.class, Integer.valueOf(NUM_BYTES_LONG));
+ }
- private boolean checkInterned;
+ private final static Object theUnsafe;
+ private final static Method objectFieldOffsetMethod;
+ private final static boolean useUnsafe, isSupportedJVM;
/**
- * Constructs this object with an AverageGuessMemoryModel and
- * checkInterned = true.
+ * Initialize constants and try to collect information about the JVM internals.
*/
- public RamUsageEstimator() {
- this(new AverageGuessMemoryModel());
+ static {
+ // Initialize empirically measured defaults. We'll modify them to the current
+ // JVM settings later on if possible.
+ int referenceSize = Constants.JRE_IS_64BIT ? 8 : 4;
+ int objectHeader = Constants.JRE_IS_64BIT ? 16 : 8;
+ // The following is objectHeader + NUM_BYTES_INT, but aligned (object alignment)
+ // so on 64 bit JVMs it'll be align(16 + 4, @8) = 24.
+ int arrayHeader = Constants.JRE_IS_64BIT ? 24 : 12;
+
+ Object unsafe = null;
+ Method objectFieldOffsetM = null;
+ boolean supportedJvm = true;
+ try {
+ final Class<?> unsafeClass = Class.forName("sun.misc.Unsafe");
+ final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ unsafe = unsafeField.get(null);
+
+ // get object reference size by getting scale factor of Object[] arrays:
+ try {
+ final Method arrayIndexScaleM = unsafeClass.getMethod("arrayIndexScale", Class.class);
+ referenceSize = ((Number) arrayIndexScaleM.invoke(unsafe, Object[].class)).intValue();
+ } catch (Exception e) {
+ // ignore
+ supportedJvm = false;
+ }
+
+ // updated best guess based on reference size:
+ objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8;
+ arrayHeader = Constants.JRE_IS_64BIT ? (8 + 2 * referenceSize) : 12;
+
+ // get the object header size:
+ // - first try out if the field offsets are not scaled (see warning in Unsafe docs)
+ // - get the object header size by getting the field offset of the first field of a dummy object
+ // If the scaling is byte-wise and unsafe is available, enable dynamic size measurement for
+ // estimateRamUsage().
+ try {
+ objectFieldOffsetM = unsafeClass.getMethod("objectFieldOffset", Field.class);
+ final Field dummy1Field = DummyTwoLongObject.class.getDeclaredField("dummy1");
+ final int ofs1 = ((Number) objectFieldOffsetM.invoke(unsafe, dummy1Field)).intValue();
+ final Field dummy2Field = DummyTwoLongObject.class.getDeclaredField("dummy2");
+ final int ofs2 = ((Number) objectFieldOffsetM.invoke(unsafe, dummy2Field)).intValue();
+ if (Math.abs(ofs2 - ofs1) == NUM_BYTES_LONG) {
+ final Field baseField = DummyOneFieldObject.class.getDeclaredField("base");
+ objectHeader = ((Number) objectFieldOffsetM.invoke(unsafe, baseField)).intValue();
+ } else {
+ // it is not safe to use Unsafe.objectFieldOffset(),
+ // as it may be scaled (see "cookie" comment in Unsafe), better use defaults
+ // and conventional size estimation:
+ objectFieldOffsetM = null;
+ supportedJvm = false;
+ }
+ } catch (Exception e) {
+ // on exception ensure useUnsafe will be set to false later:
+ objectFieldOffsetM = null;
+ supportedJvm = false;
+ }
+
+ // Get the array header size by retrieving the array base offset
+ // (offset of the first element of an array).
+ try {
+ final Method arrayBaseOffsetM = unsafeClass.getMethod("arrayBaseOffset", Class.class);
+ // we calculate that only for byte[] arrays, it's actually the same for all types:
+ arrayHeader = ((Number) arrayBaseOffsetM.invoke(unsafe, byte[].class)).intValue();
+ } catch (Exception e) {
+ // ignore
+ supportedJvm = false;
+ }
+ } catch (Exception e) {
+ // ignore
+ supportedJvm = false;
+ }
+
+ NUM_BYTES_OBJECT_REF = referenceSize;
+ NUM_BYTES_OBJECT_HEADER = objectHeader;
+ NUM_BYTES_ARRAY_HEADER = arrayHeader;
+ useUnsafe = (unsafe != null && objectFieldOffsetM != null);
+ if (useUnsafe) {
+ theUnsafe = unsafe;
+ objectFieldOffsetMethod = objectFieldOffsetM;
+ } else {
+ theUnsafe = objectFieldOffsetMethod = null;
+ }
+
+ // Try to get the object alignment (the default seems to be 8 on Hotspot,
+ // regardless of the architecture). Retrieval only works with Java 6.
+ int objectAlignment = 8;
+ if (Constants.JRE_IS_MINIMUM_JAVA6) {
+ try {
+ final Class<?> beanClazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean");
+ final Object hotSpotBean = ManagementFactory.newPlatformMXBeanProxy(
+ ManagementFactory.getPlatformMBeanServer(),
+ "com.sun.management:type=HotSpotDiagnostic",
+ beanClazz
+ );
+ final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class);
+ try {
+ final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes");
+ objectAlignment = Integer.parseInt(
+ vmOption.getClass().getMethod("getValue").invoke(vmOption).toString()
+ );
+ } catch (InvocationTargetException ite) {
+ if (!(ite.getCause() instanceof IllegalArgumentException))
+ throw ite;
+ // ignore the error completely and use default of 8 (32 bit JVMs).
+ }
+ } catch (Exception e) {
+ // ignore
+ supportedJvm = false;
+ }
+ }
+ NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment;
+
+ isSupportedJVM = supportedJvm;
}
- /**
- * @param checkInterned check if Strings are interned and don't add to size
- * if they are. Defaults to true but if you know the objects you are checking
- * won't likely contain many interned Strings, it will be faster to turn off
- * intern checking.
+ // Object with just one field to determine the object header size by getting the offset of the dummy field:
+ @SuppressWarnings("unused")
+ private static final class DummyOneFieldObject {
+ public byte base;
+ }
+
+ // Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes.
+ // Only then we can be sure that those are real, unscaled offsets:
+ @SuppressWarnings("unused")
+ private static final class DummyTwoLongObject {
+ public long dummy1, dummy2;
+ }
+
+ /**
+ * Returns true, if the current JVM is supported by {@code RamUsageEstimator}.
+ * If this method returns {@code false} you are maybe using a 3rd party Java VM
+ * that is not supporting Oracle/Sun private APIs. The memory estimates can be
+ * imprecise then (no way of detecting compressed references, alignments, etc.).
+ * Lucene still tries to use sensible defaults.
*/
- public RamUsageEstimator(boolean checkInterned) {
- this(new AverageGuessMemoryModel(), checkInterned);
+ public static boolean isSupportedJVM() {
+ return isSupportedJVM;
}
- /**
- * @param memoryModel MemoryModel to use for primitive object sizes.
+ /**
+ * Aligns an object size to be the next multiple of {@link #NUM_BYTES_OBJECT_ALIGNMENT}.
*/
- public RamUsageEstimator(MemoryModel memoryModel) {
- this(memoryModel, true);
+ public static long alignObjectSize(long size) {
+ size += (long) NUM_BYTES_OBJECT_ALIGNMENT - 1L;
+ return size - (size % NUM_BYTES_OBJECT_ALIGNMENT);
+ }
+
+ /** Returns the size in bytes of the byte[] object. */
+ public static long sizeOf(byte[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length);
+ }
+
+ /** Returns the size in bytes of the boolean[] object. */
+ public static long sizeOf(boolean[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length);
+ }
+
+ /** Returns the size in bytes of the char[] object. */
+ public static long sizeOf(char[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_CHAR * arr.length);
}
- /**
- * @param memoryModel MemoryModel to use for primitive object sizes.
- * @param checkInterned check if Strings are interned and don't add to size
- * if they are. Defaults to true but if you know the objects you are checking
- * won't likely contain many interned Strings, it will be faster to turn off
- * intern checking.
+ /** Returns the size in bytes of the short[] object. */
+ public static long sizeOf(short[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_SHORT * arr.length);
+ }
+
+ /** Returns the size in bytes of the int[] object. */
+ public static long sizeOf(int[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_INT * arr.length);
+ }
+
+ /** Returns the size in bytes of the float[] object. */
+ public static long sizeOf(float[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_FLOAT * arr.length);
+ }
+
+ /** Returns the size in bytes of the long[] object. */
+ public static long sizeOf(long[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_LONG * arr.length);
+ }
+
+ /** Returns the size in bytes of the double[] object. */
+ public static long sizeOf(double[] arr) {
+ return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_DOUBLE * arr.length);
+ }
+
+ /**
+ * Estimates the RAM usage by the given object. It will
+ * walk the object tree and sum up all referenced objects.
+ *
+ * <p><b>Resource Usage:</b> This method internally uses a set of
+ * every object seen during traversals so it does allocate memory
+ * (it isn't side-effect free). After the method exits, this memory
+ * should be GCed.</p>
*/
- public RamUsageEstimator(MemoryModel memoryModel, boolean checkInterned) {
- this.memoryModel = memoryModel;
- this.checkInterned = checkInterned;
- // Use Map rather than Set so that we can use an IdentityHashMap - not
- // seeing an IdentityHashSet
- seen = new IdentityHashMap<Object,Object>(64);
- this.refSize = memoryModel.getReferenceSize();
- this.arraySize = memoryModel.getArraySize();
- this.classSize = memoryModel.getClassSize();
+ public static long sizeOf(Object obj) {
+ final Set<Object> seen = new MapBackedSet<Object>(new IdentityHashMap<Object,Boolean>(64));
+ try {
+ return measureObjectSize(obj, seen, false);
+ } finally {
+ // Help the GC.
+ seen.clear();
+ }
}
- public long estimateRamUsage(Object obj) {
- long size = size(obj);
- seen.clear();
- return size;
+ /**
+ * Estimates a "shallow" memory usage of the given object. For arrays, this will be the
+ * memory taken by array storage (no subreferences will be followed). For objects, this
+ * will be the memory taken by the fields.
+ *
+ * JVM object alignments are also applied.
+ */
+ public static long shallowSizeOf(Object obj) {
+ if (obj == null) return 0;
+ final Class<?> clz = obj.getClass();
+ if (clz.isArray()) {
+ return measureArraySize(obj, null, false);
+ } else {
+ return shallowSizeOfInstance(clz);
+ }
}
- private long size(Object obj) {
+ /**
+ * Returns the shallow instance size in bytes an instance of the given class would occupy.
+ * This works with all conventional classes and primitive types, but not with arrays
+ * (the size then depends on the number of elements and varies from object to object).
+ * Use the array-instance methods instead.
+ *
+ * @throws IllegalArgumentException if {@code clazz} is an array class.
+ */
+ public static long shallowSizeOfInstance(Class<?> clazz) {
+ if (clazz.isArray())
+ throw new IllegalArgumentException("This method does not work with array classes.");
+ if (clazz.isPrimitive())
+ return primitiveSizes.get(clazz);
+
+ long size = NUM_BYTES_OBJECT_HEADER;
+
+ // Walk type hierarchy
+ while (clazz != null) {
+ final Field[] fields = clazz.getDeclaredFields();
+ boolean fieldFound = false;
+ for (final Field f : fields) {
+ if (Modifier.isStatic(f.getModifiers())) {
+ continue;
+ }
+
+ size = reflectFieldSize(size, f);
+ fieldFound = true;
+ }
+ if (useUnsafe && fieldFound) {
+ // no need to recurse to superclasses, as all fields are
+ // added at the end, so we won't find any larger offset
+ break;
+ }
+ clazz = clazz.getSuperclass();
+ }
+ return alignObjectSize(size);
+ }
+
+ /**
+ * Recursive descend into an object.
+ */
+ private static long measureObjectSize(Object obj, Set<Object> seen, final boolean checkInterned) {
if (obj == null) {
return 0;
}
- // interned not part of this object
- if (checkInterned && obj instanceof String
- && obj == ((String) obj).intern()) { // interned string will be eligible
- // for GC on
- // estimateRamUsage(Object) return
+
+ // *** BEGIN deprecation
+ if (checkInterned && obj instanceof String && obj == ((String) obj).intern()) {
return 0;
}
+ // *** END deprecation
// skip if we have seen before
- if (seen.containsKey(obj)) {
+ if (seen.contains(obj)) {
return 0;
}
// add to seen
- seen.put(obj, null);
+ seen.add(obj);
Class<?> clazz = obj.getClass();
if (clazz.isArray()) {
- return sizeOfArray(obj);
+ return measureArraySize(obj, seen, checkInterned);
}
- long size = 0;
+ long size = NUM_BYTES_OBJECT_HEADER;
+ long innerSize = 0L;
// walk type hierarchy
while (clazz != null) {
- Field[] fields = clazz.getDeclaredFields();
- for (int i = 0; i < fields.length; i++) {
- if (Modifier.isStatic(fields[i].getModifiers())) {
+ final Field[] fields = clazz.getDeclaredFields();
+ for (final Field f : fields) {
+ if (Modifier.isStatic(f.getModifiers())) {
continue;
}
- if (fields[i].getType().isPrimitive()) {
- size += memoryModel.getPrimitiveSize(fields[i].getType());
- } else {
- size += refSize;
- fields[i].setAccessible(true);
+ size = reflectFieldSize(size, f);
+
+ if (!f.getType().isPrimitive()) {
try {
- Object value = fields[i].get(obj);
- if (value != null) {
- size += size(value);
- }
+ f.setAccessible(true);
+ innerSize += measureObjectSize(f.get(obj), seen, checkInterned);
} catch (IllegalAccessException ex) {
- // ignore for now?
+ // this should never happen as we enable setAccessible()!
+ throw new RuntimeException("Cannot reflect instance field: " +
+ f.getDeclaringClass().getName() + "#" + f.getName(), ex);
}
}
-
}
clazz = clazz.getSuperclass();
}
- size += classSize;
- return size;
+ return alignObjectSize(size) + innerSize;
}
-
- private long sizeOfArray(Object obj) {
- int len = Array.getLength(obj);
- if (len == 0) {
- return 0;
- }
- long size = arraySize;
- Class<?> arrayElementClazz = obj.getClass().getComponentType();
- if (arrayElementClazz.isPrimitive()) {
- size += len * memoryModel.getPrimitiveSize(arrayElementClazz);
+
+ private static long reflectFieldSize(long size, final Field f) {
+ final Class<?> type = f.getType();
+ final int fsize = type.isPrimitive() ? primitiveSizes.get(type) : NUM_BYTES_OBJECT_REF;
+ if (useUnsafe) {
+ try {
+ final long offsetPlusSize =
+ ((Number) objectFieldOffsetMethod.invoke(theUnsafe, f)).longValue() + fsize;
+ return Math.max(size, offsetPlusSize);
+ } catch (IllegalAccessException ex) {
+ throw new RuntimeException("Access problem with sun.misc.Unsafe", ex);
+ } catch (InvocationTargetException ite) {
+ final Throwable cause = ite.getCause();
+ if (cause instanceof RuntimeException)
+ throw (RuntimeException) cause;
+ if (cause instanceof Error)
+ throw (Error) cause;
+ // this should never happen (Unsafe does not declare
+ // checked Exceptions for this method), but who knows?
+ throw new RuntimeException("Call to Unsafe's objectFieldOffset() throwed "+
+ "checked Exception when accessing field " +
+ f.getDeclaringClass().getName() + "#" + f.getName(), cause);
+ }
} else {
- for (int i = 0; i < len; i++) {
- size += refSize + size(Array.get(obj, i));
+ return size + fsize;
+ }
+ }
+
+ /**
+ * Return the deep size of an <code>array</code>, including
+ * sub-objects if there are any.
+ *
+ * @param seen A set of already seen objects. If <code>null</code> no references
+ * are followed and this method returns shallow size.
+ */
+ private static long measureArraySize(Object array, Set<Object> seen, final boolean checkInterned) {
+ long size = NUM_BYTES_ARRAY_HEADER;
+ final int len = Array.getLength(array);
+ if (len > 0) {
+ Class<?> arrayElementClazz = array.getClass().getComponentType();
+ if (arrayElementClazz.isPrimitive()) {
+ size += (long) len * primitiveSizes.get(arrayElementClazz);
+ } else {
+ size += (long) NUM_BYTES_OBJECT_REF * len;
+ if (seen != null) {
+ for (int i = 0; i < len; i++) {
+ size += measureObjectSize(Array.get(array, i), seen, checkInterned);
+ }
+ }
}
}
- return size;
+ return alignObjectSize(size);
}
- private static final long ONE_KB = 1024;
- private static final long ONE_MB = ONE_KB * ONE_KB;
- private static final long ONE_GB = ONE_KB * ONE_MB;
+ public static final long ONE_KB = 1024;
+ public static final long ONE_MB = ONE_KB * ONE_KB;
+ public static final long ONE_GB = ONE_KB * ONE_MB;
/**
- * Return good default units based on byte size.
+ * Returns <code>size</code> in human-readable units (GB, MB, KB or bytes).
*/
- public static String humanReadableUnits(long bytes, DecimalFormat df) {
- String newSizeAndUnits;
+ public static String humanReadableUnits(long bytes) {
+ return humanReadableUnits(bytes,
+ new DecimalFormat("0.#", new DecimalFormatSymbols(Locale.ENGLISH)));
+ }
+ /**
+ * Returns <code>size</code> in human-readable units (GB, MB, KB or bytes).
+ */
+ public static String humanReadableUnits(long bytes, DecimalFormat df) {
if (bytes / ONE_GB > 0) {
- newSizeAndUnits = String.valueOf(df.format((float) bytes / ONE_GB))
- + " GB";
+ return df.format((float) bytes / ONE_GB) + " GB";
} else if (bytes / ONE_MB > 0) {
- newSizeAndUnits = String.valueOf(df.format((float) bytes / ONE_MB))
- + " MB";
+ return df.format((float) bytes / ONE_MB) + " MB";
} else if (bytes / ONE_KB > 0) {
- newSizeAndUnits = String.valueOf(df.format((float) bytes / ONE_KB))
- + " KB";
+ return df.format((float) bytes / ONE_KB) + " KB";
} else {
- newSizeAndUnits = String.valueOf(bytes) + " bytes";
+ return bytes + " bytes";
}
+ }
- return newSizeAndUnits;
+ // deprecated API (will be removed in 4.0):
+ private final boolean checkInterned;
+ private final Set<Object> seen =
+ new MapBackedSet<Object>(new IdentityHashMap<Object,Boolean>(64));
+
+ /** Creates a new instance of {@code RamUsageEstimator} with intern checking
+ * enabled. Don't ever use this method, as intern checking is deprecated,
+ * because it is not free of side-effects and strains the garbage collector
+ * additionally.
+ * @deprecated Don't create instances of this class, instead use the static
+ * {@link #sizeOf(Object)} method that has no intern checking, too.
+ */
+ @Deprecated
+ public RamUsageEstimator() {
+ this(true);
+ }
+
+ /** Creates a new instance of {@code RamUsageEstimator}.
+ * @param checkInterned check if Strings are interned and don't add to size
+ * if they are. Defaults to true but if you know the objects you are checking
+ * won't likely contain many interned Strings, it will be faster to turn off
+ * intern checking. Intern checking is deprecated altogether, as it is not free
+ * of side-effects and strains the garbage collector additionally.
+ * @deprecated Don't create instances of this class, instead use the static
+ * {@link #sizeOf(Object)} method.
+ */
+ @Deprecated
+ public RamUsageEstimator(boolean checkInterned) {
+ this.checkInterned = checkInterned;
+ }
+
+ /** Creates a new istance of {@code RamUsageEstimator}.
+ * @deprecated Don't create instances of this class, instead use the static
+ * {@link #sizeOf(Object)} method.
+ * @see #sizeOf(Object)
+ */
+ @Deprecated
+ public long estimateRamUsage(Object obj) {
+ try {
+ return measureObjectSize(obj, seen, checkInterned);
+ } finally {
+ // Help the GC.
+ seen.clear();
+ }
}
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java Sun Mar 18 16:12:11 2012
@@ -78,8 +78,7 @@ class Direct16 extends PackedInts.Reader
}
public long ramBytesUsed() {
- return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER +
- values.length * RamUsageEstimator.NUM_BYTES_SHORT;
+ return RamUsageEstimator.sizeOf(values);
}
public void clear() {
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java Sun Mar 18 16:12:11 2012
@@ -74,8 +74,7 @@ class Direct32 extends PackedInts.Reader
}
public long ramBytesUsed() {
- return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER +
- values.length * RamUsageEstimator.NUM_BYTES_INT;
+ return RamUsageEstimator.sizeOf(values);
}
public void clear() {
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java Sun Mar 18 16:12:11 2012
@@ -70,8 +70,7 @@ class Direct64 extends PackedInts.Reader
}
public long ramBytesUsed() {
- return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER +
- values.length * RamUsageEstimator.NUM_BYTES_LONG;
+ return RamUsageEstimator.sizeOf(values);
}
public void clear() {
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java Sun Mar 18 16:12:11 2012
@@ -79,7 +79,7 @@ class Direct8 extends PackedInts.ReaderI
}
public long ramBytesUsed() {
- return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + values.length;
+ return RamUsageEstimator.sizeOf(values);
}
public void clear() {
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java Sun Mar 18 16:12:11 2012
@@ -222,7 +222,6 @@ class Packed32 extends PackedInts.Reader
}
public long ramBytesUsed() {
- return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER
- + blocks.length * RamUsageEstimator.NUM_BYTES_INT;
+ return RamUsageEstimator.sizeOf(blocks);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java Sun Mar 18 16:12:11 2012
@@ -208,8 +208,7 @@ class Packed64 extends PackedInts.Reader
}
public long ramBytesUsed() {
- return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER
- + blocks.length * RamUsageEstimator.NUM_BYTES_LONG;
+ return RamUsageEstimator.sizeOf(blocks);
}
public void clear() {
Copied: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java (from r1302133, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java?p2=lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java&p1=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java&r1=1302133&r2=1302152&rev=1302152&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/StressRamUsageEstimator.java Sun Mar 18 16:12:11 2012
@@ -5,7 +5,6 @@ import java.lang.management.ManagementFa
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Random;
@@ -115,7 +114,9 @@ public class StressRamUsageEstimator ext
// Make another batch of objects.
Object[] seg = new Object[10000];
- all = Arrays.copyOf(all, all.length + 1);
+ Object[][] all2 = new Object [all.length + 1][];
+ System.arraycopy(all, 0, all2, 0, all.length);
+ all = all2;
all[all.length - 1] = seg;
for (int i = 0; i < seg.length; i++) {
seg[i] = new byte[random.nextInt(7)];
Modified: lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java?rev=1302152&r1=1302151&r2=1302152&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java (original)
+++ lucene/dev/branches/branch_3x/lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java Sun Mar 18 16:12:11 2012
@@ -1,5 +1,9 @@
package org.apache.lucene.util;
+import static org.apache.lucene.util.RamUsageEstimator.*;
+
+import java.util.Random;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -17,28 +21,93 @@ package org.apache.lucene.util;
* limitations under the License.
*/
-import org.apache.lucene.util.LuceneTestCase;
-
public class TestRamUsageEstimator extends LuceneTestCase {
public void testBasic() {
- RamUsageEstimator rue = new RamUsageEstimator();
- rue.estimateRamUsage("test str");
-
- rue.estimateRamUsage("test strin");
+ assertTrue(sizeOf(new String("test strin")) > shallowSizeOfInstance(String.class));
Holder holder = new Holder();
holder.holder = new Holder("string2", 5000L);
- rue.estimateRamUsage(holder);
+ assertTrue(sizeOf(holder) > shallowSizeOfInstance(Holder.class));
+ assertTrue(sizeOf(holder) > sizeOf(holder.holder));
+ assertTrue(shallowSizeOfInstance(HolderSubclass.class) >= shallowSizeOfInstance(Holder.class));
+ assertEquals(shallowSizeOfInstance(Holder.class), shallowSizeOfInstance(HolderSubclass2.class));
+
String[] strings = new String[]{new String("test strin"), new String("hollow"), new String("catchmaster")};
- rue.estimateRamUsage(strings);
+ assertTrue(sizeOf(strings) > shallowSizeOf(strings));
+ }
+
+ public void testStaticOverloads() {
+ Random rnd = random;
+
+ {
+ byte[] array = new byte [rnd.nextInt(1024)];
+ assertEquals(sizeOf(array), sizeOf((Object) array));
+ }
+
+ {
+ boolean[] array = new boolean [rnd.nextInt(1024)];
+ assertEquals(sizeOf(array), sizeOf((Object) array));
+ }
+
+ {
+ char[] array = new char [rnd.nextInt(1024)];
+ assertEquals(sizeOf(array), sizeOf((Object) array));
+ }
+
+ {
+ short[] array = new short [rnd.nextInt(1024)];
+ assertEquals(sizeOf(array), sizeOf((Object) array));
+ }
+
+ {
+ int[] array = new int [rnd.nextInt(1024)];
+ assertEquals(sizeOf(array), sizeOf((Object) array));
+ }
+
+ {
+ float[] array = new float [rnd.nextInt(1024)];
+ assertEquals(sizeOf(array), sizeOf((Object) array));
+ }
+
+ {
+ long[] array = new long [rnd.nextInt(1024)];
+ assertEquals(sizeOf(array), sizeOf((Object) array));
+ }
+
+ {
+ double[] array = new double [rnd.nextInt(1024)];
+ assertEquals(sizeOf(array), sizeOf((Object) array));
+ }
+ }
+
+ public void testReferenceSize() {
+ if (!isSupportedJVM()) {
+ System.err.println("WARN: Your JVM does not support the Oracle/Sun extensions (sun.misc.Unsafe),");
+ System.err.println("so the memory estimates may be inprecise.");
+ System.err.println("Please report this to the Lucene mailing list, noting your JVM version: " +
+ Constants.JAVA_VENDOR + " " + Constants.JAVA_VERSION);
+ }
+ if (VERBOSE) {
+ System.out.println("This JVM is 64bit: " + Constants.JRE_IS_64BIT);
+ System.out.println("Reference size in this JVM: " + NUM_BYTES_OBJECT_REF);
+ System.out.println("Object header size in this JVM: " + NUM_BYTES_OBJECT_HEADER);
+ System.out.println("Array header size in this JVM: " + NUM_BYTES_ARRAY_HEADER);
+ System.out.println("Object alignment in this JVM: " + NUM_BYTES_OBJECT_ALIGNMENT);
+ }
+ assertTrue(NUM_BYTES_OBJECT_REF == 4 || NUM_BYTES_OBJECT_REF == 8);
+ if (!Constants.JRE_IS_64BIT) {
+ assertEquals("For 32bit JVMs, reference size must always be 4", 4, NUM_BYTES_OBJECT_REF);
+ }
}
- private static final class Holder {
+ @SuppressWarnings("unused")
+ private static class Holder {
long field1 = 5000L;
String name = "name";
Holder holder;
+ long field2, field3, field4;
Holder() {
}
@@ -48,4 +117,14 @@ public class TestRamUsageEstimator exten
this.field1 = field1;
}
}
+
+ @SuppressWarnings("unused")
+ private static class HolderSubclass extends Holder {
+ byte foo;
+ int bar;
+ }
+
+ private static class HolderSubclass2 extends Holder {
+ // empty, only inherits all fields -> size should be identical to superclass
+ }
}