You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2016/10/02 10:16:51 UTC
[20/49] lucenenet git commit: Ported StressRamUsageEstimator &
TestRamUsageEstimator + fixed bugs in RamUsageEstimator
Ported StressRamUsageEstimator & TestRamUsageEstimator + fixed bugs in RamUsageEstimator
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/b1203e6b
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/b1203e6b
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/b1203e6b
Branch: refs/heads/master
Commit: b1203e6b4cb9551fe09999114890e74e77cb6937
Parents: ef889ce
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Sat Sep 10 17:53:32 2016 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Sat Sep 10 17:53:32 2016 +0700
----------------------------------------------------------------------
src/Lucene.Net.Core/Util/RamUsageEstimator.cs | 53 +++++++-----
src/Lucene.Net.Tests/Lucene.Net.Tests.csproj | 2 +
.../core/Util/StressRamUsageEstimator.cs | 62 +++++---------
.../core/Util/TestRamUsageEstimator.cs | 85 ++++++++++----------
4 files changed, 98 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b1203e6b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
index d3f289b..07f6037 100644
--- a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
+++ b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
@@ -111,7 +111,8 @@ namespace Lucene.Net.Util
int arrayHeader = Constants.JRE_IS_64BIT ? 24 : 12;
int objectAlignment = Constants.JRE_IS_64BIT ? 8 : 4;
- /* LUCENE-TODO
+
+ /* LUCENENET TODO
Type unsafeClass = null;
object tempTheUnsafe = null;
@@ -138,12 +139,15 @@ namespace Lucene.Net.Util
{
// ignore.
}
+ */
- // "best guess" based on reference size. We will attempt to modify
- // these to exact values if there is supported infrastructure.
- objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8;
+ // "best guess" based on reference size. We will attempt to modify
+ // these to exact values if there is supported infrastructure.
+ objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8;
arrayHeader = Constants.JRE_IS_64BIT ? (8 + 2 * referenceSize) : 12;
+ /* LUCENENET TODO
+
// get the object header size:
// - first try out if the field offsets are not scaled (see warning in Unsafe docs)
// - get the object header size by getting the field offset of the first field of a dummy object
@@ -187,7 +191,7 @@ namespace Lucene.Net.Util
NUM_BYTES_OBJECT_HEADER = objectHeader;
NUM_BYTES_ARRAY_HEADER = arrayHeader;
- /* LUCENE-TODO
+ /* LUCENENET TODO
// Try to get the object alignment (the default seems to be 8 on Hotspot,
// regardless of the architecture).
int objectAlignment = 8;
@@ -207,6 +211,7 @@ namespace Lucene.Net.Util
// Ignore.
}
*/
+
NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment;
JVM_INFO_STRING = "[JVM: " + Constants.JVM_NAME + ", " + Constants.JVM_VERSION + ", " + Constants.JVM_VENDOR + ", " + Constants.JAVA_VENDOR + ", " + Constants.JAVA_VERSION + "]";
@@ -215,7 +220,7 @@ namespace Lucene.Net.Util
/// <summary>
/// A handle to <code>sun.misc.Unsafe</code>.
/// </summary>
- private static readonly object TheUnsafe;
+ //private static readonly object TheUnsafe;
/// <summary>
/// A handle to <code>sun.misc.Unsafe#fieldOffset(Field)</code>.
@@ -237,18 +242,18 @@ namespace Lucene.Net.Util
}
}
- // Object with just one field to determine the object header size by getting the offset of the dummy field:
- private sealed class DummyOneFieldObject
- {
- public sbyte @base;
- }
+ //// Object with just one field to determine the object header size by getting the offset of the dummy field:
+ //private sealed class DummyOneFieldObject
+ //{
+ // public sbyte @base;
+ //}
- // Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes.
- // Only then we can be sure that those are real, unscaled offsets:
- private sealed class DummyTwoLongObject
- {
- public long Dummy1, Dummy2;
- }
+ //// Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes.
+ //// Only then we can be sure that those are real, unscaled offsets:
+ //private sealed class DummyTwoLongObject
+ //{
+ // public long Dummy1, Dummy2;
+ //}
/// <summary>
/// Aligns an object size to be the next multiple of <seealso cref="#NUM_BYTES_OBJECT_ALIGNMENT"/>.
@@ -376,7 +381,12 @@ namespace Lucene.Net.Util
// Walk type hierarchy
for (; clazz != null; clazz = clazz.BaseType)
{
- FieldInfo[] fields = clazz.GetFields(BindingFlags.Public);
+ FieldInfo[] fields = clazz.GetFields(
+ BindingFlags.Instance |
+ BindingFlags.NonPublic |
+ BindingFlags.Public |
+ BindingFlags.DeclaredOnly |
+ BindingFlags.Static);
foreach (FieldInfo f in fields)
{
if (!f.IsStatic)
@@ -525,7 +535,12 @@ namespace Lucene.Net.Util
List<FieldInfo> referenceFields = new List<FieldInfo>(32);
for (Type c = clazz; c != null; c = c.BaseType)
{
- FieldInfo[] fields = c.GetFields(BindingFlags.Public | BindingFlags.Instance);
+ FieldInfo[] fields = c.GetFields(
+ BindingFlags.Instance |
+ BindingFlags.NonPublic |
+ BindingFlags.Public |
+ BindingFlags.DeclaredOnly |
+ BindingFlags.Static);
foreach (FieldInfo f in fields)
{
if (!f.IsStatic)
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b1203e6b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
index 955d5bc..03f336c 100644
--- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
+++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
@@ -470,6 +470,7 @@
<SubType>Code</SubType>
</Compile>
<Compile Include="core\Util\Packed\TestPackedInts.cs" />
+ <Compile Include="core\Util\StressRamUsageEstimator.cs" />
<Compile Include="core\Util\Test2BPagedBytes.cs">
<SubType>Code</SubType>
</Compile>
@@ -562,6 +563,7 @@
<Compile Include="core\Util\TestQueryBuilder.cs">
<SubType>Code</SubType>
</Compile>
+ <Compile Include="core\Util\TestRamUsageEstimator.cs" />
<Compile Include="core\Util\TestRamUsageEstimatorOnWildAnimals.cs">
<SubType>Code</SubType>
</Compile>
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b1203e6b/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs b/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs
index 0bc9f75..c0b8533 100644
--- a/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs
+++ b/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs
@@ -1,10 +1,10 @@
-using System;
-using System.Collections.Generic;
+using Lucene.Net.Support;
using NUnit.Framework;
+using System;
+using System.Globalization;
namespace Lucene.Net.Util
{
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -50,8 +50,6 @@ namespace Lucene.Net.Util
[Test]
public virtual void TestChainedEstimation()
{
- MemoryMXBean memoryMXBean = ManagementFactory.MemoryMXBean;
-
Random rnd = Random();
Entry first = new Entry();
try
@@ -59,9 +57,9 @@ namespace Lucene.Net.Util
while (true)
{
// Check the current memory consumption and provide the estimate.
- long jvmUsed = memoryMXBean.HeapMemoryUsage.Used;
- long estimated = RamUsageEstimator.sizeOf(first);
- Console.WriteLine(string.format(Locale.ROOT, "%10d, %10d", jvmUsed, estimated));
+ long jvmUsed = GC.GetTotalMemory(false);
+ long estimated = RamUsageEstimator.SizeOf(first);
+ Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0:0000000000}, {1:0000000000}", jvmUsed, estimated));
// Make a batch of objects.
for (int i = 0; i < 5000; i++)
@@ -82,52 +80,48 @@ namespace Lucene.Net.Util
[Test]
public virtual void TestLargeSetOfByteArrays()
{
- MemoryMXBean memoryMXBean = ManagementFactory.MemoryMXBean;
-
CauseGc();
- long before = memoryMXBean.HeapMemoryUsage.Used;
+ long before = GC.GetTotalMemory(false);
object[] all = new object[1000000];
for (int i = 0; i < all.Length; i++)
{
all[i] = new sbyte[Random().Next(3)];
}
CauseGc();
- long after = memoryMXBean.HeapMemoryUsage.Used;
- Console.WriteLine("mx: " + RamUsageEstimator.humanReadableUnits(after - before));
- Console.WriteLine("rue: " + RamUsageEstimator.humanReadableUnits(ShallowSizeOf(all)));
+ long after = GC.GetTotalMemory(false);
+ Console.WriteLine("mx: " + RamUsageEstimator.HumanReadableUnits(after - before));
+ Console.WriteLine("rue: " + RamUsageEstimator.HumanReadableUnits(ShallowSizeOf(all)));
Guard = all;
}
private long ShallowSizeOf(object[] all)
{
- long s = RamUsageEstimator.shallowSizeOf(all);
+ long s = RamUsageEstimator.ShallowSizeOf(all);
foreach (object o in all)
{
- s += RamUsageEstimator.shallowSizeOf(o);
+ s += RamUsageEstimator.ShallowSizeOf(o);
}
return s;
}
private long ShallowSizeOf(object[][] all)
{
- long s = RamUsageEstimator.shallowSizeOf(all);
+ long s = RamUsageEstimator.ShallowSizeOf(all);
foreach (object[] o in all)
{
- s += RamUsageEstimator.shallowSizeOf(o);
+ s += RamUsageEstimator.ShallowSizeOf(o);
foreach (object o2 in o)
{
- s += RamUsageEstimator.shallowSizeOf(o2);
+ s += RamUsageEstimator.ShallowSizeOf(o2);
}
}
return s;
}
- [Test]
+ [Test, Timeout(180000)]
public virtual void TestSimpleByteArrays()
{
- MemoryMXBean memoryMXBean = ManagementFactory.MemoryMXBean;
-
object[][] all = new object[0][];
try
{
@@ -135,18 +129,18 @@ namespace Lucene.Net.Util
{
// Check the current memory consumption and provide the estimate.
CauseGc();
- MemoryUsage mu = memoryMXBean.HeapMemoryUsage;
+
long estimated = ShallowSizeOf(all);
if (estimated > 50 * RamUsageEstimator.ONE_MB)
{
break;
}
- Console.WriteLine(string.format(Locale.ROOT, "%10s\t%10s\t%10s", RamUsageEstimator.humanReadableUnits(mu.Used), RamUsageEstimator.humanReadableUnits(mu.Max), RamUsageEstimator.humanReadableUnits(estimated)));
+ Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0}\t{1}\t{2}", RamUsageEstimator.HumanReadableUnits(GC.GetTotalMemory(false)).PadLeft(10, ' '), RamUsageEstimator.HumanReadableUnits(GC.MaxGeneration).PadLeft(10, ' '), RamUsageEstimator.HumanReadableUnits(estimated).PadLeft(10, ' ')));
// Make another batch of objects.
object[] seg = new object[10000];
- all = Arrays.copyOf(all, all.Length + 1);
+ all = Arrays.CopyOf(all, all.Length + 1);
all[all.Length - 1] = seg;
for (int i = 0; i < seg.Length; i++)
{
@@ -166,23 +160,7 @@ namespace Lucene.Net.Util
/// </summary>
private void CauseGc()
{
- IList<GarbageCollectorMXBean> garbageCollectorMXBeans = ManagementFactory.GarbageCollectorMXBeans;
- IList<long?> ccounts = new List<long?>();
- foreach (GarbageCollectorMXBean g in garbageCollectorMXBeans)
- {
- ccounts.Add(g.CollectionCount);
- }
- IList<long?> ccounts2 = new List<long?>();
- do
- {
- System.gc();
- ccounts.Clear();
- foreach (GarbageCollectorMXBean g in garbageCollectorMXBeans)
- {
- ccounts2.Add(g.CollectionCount);
- }
- } while (ccounts2.Equals(ccounts));
+ GC.Collect();
}
}
-
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b1203e6b/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs b/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs
index 113e730..b08739d 100644
--- a/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs
+++ b/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs
@@ -1,10 +1,8 @@
-using System;
-using Lucene.Net.Util;
using NUnit.Framework;
+using System;
namespace Lucene.Net.Util
{
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -25,22 +23,22 @@ namespace Lucene.Net.Util
[TestFixture]
public class TestRamUsageEstimator : LuceneTestCase
{
- /*[Test]
+ [Test]
public virtual void TestSanity()
{
- Assert.IsTrue(sizeof(new string("test string")) > ShallowSizeOfInstance(typeof(string)));
+ Assert.IsTrue(RamUsageEstimator.SizeOf("test string") > RamUsageEstimator.ShallowSizeOfInstance(typeof(string)));
Holder holder = new Holder();
holder.Holder_Renamed = new Holder("string2", 5000L);
- Assert.IsTrue(sizeof(holder) > shallowSizeOfInstance(typeof(Holder)));
- Assert.IsTrue(sizeof(holder) > sizeof(holder.Holder_Renamed));
+ Assert.IsTrue(RamUsageEstimator.SizeOf(holder) > RamUsageEstimator.ShallowSizeOfInstance(typeof(Holder)));
+ Assert.IsTrue(RamUsageEstimator.SizeOf(holder) > RamUsageEstimator.SizeOf(holder.Holder_Renamed));
- Assert.IsTrue(shallowSizeOfInstance(typeof(HolderSubclass)) >= shallowSizeOfInstance(typeof(Holder)));
- Assert.IsTrue(shallowSizeOfInstance(typeof(Holder)) == shallowSizeOfInstance(typeof(HolderSubclass2)));
+ Assert.IsTrue(RamUsageEstimator.ShallowSizeOfInstance(typeof(HolderSubclass)) >= RamUsageEstimator.ShallowSizeOfInstance(typeof(Holder)));
+ Assert.IsTrue(RamUsageEstimator.ShallowSizeOfInstance(typeof(Holder)) == RamUsageEstimator.ShallowSizeOfInstance(typeof(HolderSubclass2)));
- string[] strings = new string[] { new string("test string"), new string("hollow"), new string("catchmaster") };
- Assert.IsTrue(sizeof(strings) > shallowSizeOf(strings));
- }*/
+ string[] strings = new string[] { "test string", "hollow", "catchmaster" };
+ Assert.IsTrue(RamUsageEstimator.SizeOf(strings) > RamUsageEstimator.ShallowSizeOf(strings));
+ }
[Test]
public virtual void TestStaticOverloads()
@@ -48,73 +46,74 @@ namespace Lucene.Net.Util
Random rnd = Random();
{
sbyte[] array = new sbyte[rnd.Next(1024)];
- Assert.AreEqual(sizeof(array), sizeof((object)array));
+ assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
}
{
bool[] array = new bool[rnd.Next(1024)];
- Assert.AreEqual(sizeof(array), sizeof((object)array));
+ assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
}
{
char[] array = new char[rnd.Next(1024)];
- Assert.AreEqual(sizeof(array), sizeof((object)array));
+ assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
}
{
short[] array = new short[rnd.Next(1024)];
- Assert.AreEqual(sizeof(array), sizeof((object)array));
+ assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
}
{
int[] array = new int[rnd.Next(1024)];
- Assert.AreEqual(sizeof(array), sizeof((object)array));
+ assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
}
{
float[] array = new float[rnd.Next(1024)];
- Assert.AreEqual(sizeof(array), sizeof((object)array));
+ assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
}
{
long[] array = new long[rnd.Next(1024)];
- Assert.AreEqual(sizeof(array), sizeof((object)array));
+ assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
}
{
double[] array = new double[rnd.Next(1024)];
- Assert.AreEqual(sizeof(array), sizeof((object)array));
+ assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
}
}
- /* LUCENE PORT: The JVM is obviously not relevant to our purposes
- [Test]
+
+ [Test]
public virtual void TestReferenceSize()
{
- if (!SupportedJVM)
- {
- Console.Error.WriteLine("WARN: Your JVM does not support certain Oracle/Sun extensions.");
- Console.Error.WriteLine(" Memory estimates may be inaccurate.");
- Console.Error.WriteLine(" Please report this to the Lucene mailing list.");
- Console.Error.WriteLine("JVM version: " + RamUsageEstimator.JVM_INFO_STRING);
- Console.Error.WriteLine("UnsupportedFeatures:");
- foreach (JvmFeature f in RamUsageEstimator.UnsupportedFeatures)
- {
- Console.Error.Write(" - " + f.ToString());
- if (f == RamUsageEstimator.JvmFeature.OBJECT_ALIGNMENT)
- {
- Console.Error.Write("; Please note: 32bit Oracle/Sun VMs don't allow exact OBJECT_ALIGNMENT retrieval, this is a known issue.");
- }
- Console.Error.WriteLine();
- }
- }
-
- Assert.IsTrue(NUM_BYTES_OBJECT_REF == 4 || NUM_BYTES_OBJECT_REF == 8);
+ //LUCENENET: The JVM is obviously not relevant to our purposes
+ //if (!SupportedJVM)
+ //{
+ // Console.Error.WriteLine("WARN: Your JVM does not support certain Oracle/Sun extensions.");
+ // Console.Error.WriteLine(" Memory estimates may be inaccurate.");
+ // Console.Error.WriteLine(" Please report this to the Lucene mailing list.");
+ // Console.Error.WriteLine("JVM version: " + RamUsageEstimator.JVM_INFO_STRING);
+ // Console.Error.WriteLine("UnsupportedFeatures:");
+ // foreach (JvmFeature f in RamUsageEstimator.UnsupportedFeatures)
+ // {
+ // Console.Error.Write(" - " + f.ToString());
+ // if (f == RamUsageEstimator.JvmFeature.OBJECT_ALIGNMENT)
+ // {
+ // Console.Error.Write("; Please note: 32bit Oracle/Sun VMs don't allow exact OBJECT_ALIGNMENT retrieval, this is a known issue.");
+ // }
+ // Console.Error.WriteLine();
+ // }
+ //}
+
+ Assert.IsTrue(RamUsageEstimator.NUM_BYTES_OBJECT_REF == 4 || RamUsageEstimator.NUM_BYTES_OBJECT_REF == 8);
if (!Constants.JRE_IS_64BIT)
{
- Assert.AreEqual("For 32bit JVMs, reference size must always be 4?", 4, NUM_BYTES_OBJECT_REF);
+ assertEquals("For 32bit JVMs, reference size must always be 4?", 4, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
}
- }*/
+ }
private class Holder
{