You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2016/10/02 10:16:51 UTC

[20/49] lucenenet git commit: Ported StressRamUsageEstimator & TestRamUsageEstimator + fixed bugs in RamUsageEstimator

Ported StressRamUsageEstimator & TestRamUsageEstimator + fixed bugs in RamUsageEstimator


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/b1203e6b
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/b1203e6b
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/b1203e6b

Branch: refs/heads/master
Commit: b1203e6b4cb9551fe09999114890e74e77cb6937
Parents: ef889ce
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Sat Sep 10 17:53:32 2016 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Sat Sep 10 17:53:32 2016 +0700

----------------------------------------------------------------------
 src/Lucene.Net.Core/Util/RamUsageEstimator.cs   | 53 +++++++-----
 src/Lucene.Net.Tests/Lucene.Net.Tests.csproj    |  2 +
 .../core/Util/StressRamUsageEstimator.cs        | 62 +++++---------
 .../core/Util/TestRamUsageEstimator.cs          | 85 ++++++++++----------
 4 files changed, 98 insertions(+), 104 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b1203e6b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
index d3f289b..07f6037 100644
--- a/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
+++ b/src/Lucene.Net.Core/Util/RamUsageEstimator.cs
@@ -111,7 +111,8 @@ namespace Lucene.Net.Util
             int arrayHeader = Constants.JRE_IS_64BIT ? 24 : 12;
             int objectAlignment = Constants.JRE_IS_64BIT ? 8 : 4;
 
-            /* LUCENE-TODO
+
+            /* LUCENENET TODO
 
 		    Type unsafeClass = null;
 		    object tempTheUnsafe = null;
@@ -138,12 +139,15 @@ namespace Lucene.Net.Util
 		    {
 		      // ignore.
 		    }
+            */
 
-		    // "best guess" based on reference size. We will attempt to modify
-		    // these to exact values if there is supported infrastructure.
-		    objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8;
+            // "best guess" based on reference size. We will attempt to modify
+            // these to exact values if there is supported infrastructure.
+            objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8;
 		    arrayHeader = Constants.JRE_IS_64BIT ? (8 + 2 * referenceSize) : 12;
 
+            /* LUCENENET TODO
+             
 		    // get the object header size:
 		    // - first try out if the field offsets are not scaled (see warning in Unsafe docs)
 		    // - get the object header size by getting the field offset of the first field of a dummy object
@@ -187,7 +191,7 @@ namespace Lucene.Net.Util
             NUM_BYTES_OBJECT_HEADER = objectHeader;
             NUM_BYTES_ARRAY_HEADER = arrayHeader;
 
-            /* LUCENE-TODO
+            /* LUCENENET TODO
           // Try to get the object alignment (the default seems to be 8 on Hotspot,
           // regardless of the architecture).
           int objectAlignment = 8;
@@ -207,6 +211,7 @@ namespace Lucene.Net.Util
             // Ignore.
           }
             */
+
             NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment;
 
             JVM_INFO_STRING = "[JVM: " + Constants.JVM_NAME + ", " + Constants.JVM_VERSION + ", " + Constants.JVM_VENDOR + ", " + Constants.JAVA_VENDOR + ", " + Constants.JAVA_VERSION + "]";
@@ -215,7 +220,7 @@ namespace Lucene.Net.Util
         /// <summary>
         /// A handle to <code>sun.misc.Unsafe</code>.
         /// </summary>
-        private static readonly object TheUnsafe;
+        //private static readonly object TheUnsafe;
 
         /// <summary>
         /// A handle to <code>sun.misc.Unsafe#fieldOffset(Field)</code>.
@@ -237,18 +242,18 @@ namespace Lucene.Net.Util
             }
         }
 
-        // Object with just one field to determine the object header size by getting the offset of the dummy field:
-        private sealed class DummyOneFieldObject
-        {
-            public sbyte @base;
-        }
+        //// Object with just one field to determine the object header size by getting the offset of the dummy field:
+        //private sealed class DummyOneFieldObject
+        //{
+        //    public sbyte @base;
+        //}
 
-        // Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes.
-        // Only then we can be sure that those are real, unscaled offsets:
-        private sealed class DummyTwoLongObject
-        {
-            public long Dummy1, Dummy2;
-        }
+        //// Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes.
+        //// Only then we can be sure that those are real, unscaled offsets:
+        //private sealed class DummyTwoLongObject
+        //{
+        //    public long Dummy1, Dummy2;
+        //}
 
         /// <summary>
         /// Aligns an object size to be the next multiple of <seealso cref="#NUM_BYTES_OBJECT_ALIGNMENT"/>.
@@ -376,7 +381,12 @@ namespace Lucene.Net.Util
             // Walk type hierarchy
             for (; clazz != null; clazz = clazz.BaseType)
             {
-                FieldInfo[] fields = clazz.GetFields(BindingFlags.Public);
+                FieldInfo[] fields = clazz.GetFields(
+                    BindingFlags.Instance | 
+                    BindingFlags.NonPublic | 
+                    BindingFlags.Public | 
+                    BindingFlags.DeclaredOnly | 
+                    BindingFlags.Static);
                 foreach (FieldInfo f in fields)
                 {
                     if (!f.IsStatic)
@@ -525,7 +535,12 @@ namespace Lucene.Net.Util
             List<FieldInfo> referenceFields = new List<FieldInfo>(32);
             for (Type c = clazz; c != null; c = c.BaseType)
             {
-                FieldInfo[] fields = c.GetFields(BindingFlags.Public | BindingFlags.Instance);
+                FieldInfo[] fields = c.GetFields(
+                    BindingFlags.Instance | 
+                    BindingFlags.NonPublic | 
+                    BindingFlags.Public | 
+                    BindingFlags.DeclaredOnly | 
+                    BindingFlags.Static);
                 foreach (FieldInfo f in fields)
                 {
                     if (!f.IsStatic)

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b1203e6b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
index 955d5bc..03f336c 100644
--- a/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
+++ b/src/Lucene.Net.Tests/Lucene.Net.Tests.csproj
@@ -470,6 +470,7 @@
       <SubType>Code</SubType>
     </Compile>
     <Compile Include="core\Util\Packed\TestPackedInts.cs" />
+    <Compile Include="core\Util\StressRamUsageEstimator.cs" />
     <Compile Include="core\Util\Test2BPagedBytes.cs">
       <SubType>Code</SubType>
     </Compile>
@@ -562,6 +563,7 @@
     <Compile Include="core\Util\TestQueryBuilder.cs">
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="core\Util\TestRamUsageEstimator.cs" />
     <Compile Include="core\Util\TestRamUsageEstimatorOnWildAnimals.cs">
       <SubType>Code</SubType>
     </Compile>

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b1203e6b/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs b/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs
index 0bc9f75..c0b8533 100644
--- a/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs
+++ b/src/Lucene.Net.Tests/core/Util/StressRamUsageEstimator.cs
@@ -1,10 +1,10 @@
-using System;
-using System.Collections.Generic;
+using Lucene.Net.Support;
 using NUnit.Framework;
+using System;
+using System.Globalization;
 
 namespace Lucene.Net.Util
 {
-
     /*
      * Licensed to the Apache Software Foundation (ASF) under one or more
      * contributor license agreements.  See the NOTICE file distributed with
@@ -50,8 +50,6 @@ namespace Lucene.Net.Util
         [Test]
         public virtual void TestChainedEstimation()
         {
-            MemoryMXBean memoryMXBean = ManagementFactory.MemoryMXBean;
-
             Random rnd = Random();
             Entry first = new Entry();
             try
@@ -59,9 +57,9 @@ namespace Lucene.Net.Util
                 while (true)
                 {
                     // Check the current memory consumption and provide the estimate.
-                    long jvmUsed = memoryMXBean.HeapMemoryUsage.Used;
-                    long estimated = RamUsageEstimator.sizeOf(first);
-                    Console.WriteLine(string.format(Locale.ROOT, "%10d, %10d", jvmUsed, estimated));
+                    long jvmUsed = GC.GetTotalMemory(false);
+                    long estimated = RamUsageEstimator.SizeOf(first);
+                    Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0:0000000000}, {1:0000000000}", jvmUsed, estimated));
 
                     // Make a batch of objects.
                     for (int i = 0; i < 5000; i++)
@@ -82,52 +80,48 @@ namespace Lucene.Net.Util
         [Test]
         public virtual void TestLargeSetOfByteArrays()
         {
-            MemoryMXBean memoryMXBean = ManagementFactory.MemoryMXBean;
-
             CauseGc();
-            long before = memoryMXBean.HeapMemoryUsage.Used;
+            long before = GC.GetTotalMemory(false);
             object[] all = new object[1000000];
             for (int i = 0; i < all.Length; i++)
             {
                 all[i] = new sbyte[Random().Next(3)];
             }
             CauseGc();
-            long after = memoryMXBean.HeapMemoryUsage.Used;
-            Console.WriteLine("mx:  " + RamUsageEstimator.humanReadableUnits(after - before));
-            Console.WriteLine("rue: " + RamUsageEstimator.humanReadableUnits(ShallowSizeOf(all)));
+            long after = GC.GetTotalMemory(false);
+            Console.WriteLine("mx:  " + RamUsageEstimator.HumanReadableUnits(after - before));
+            Console.WriteLine("rue: " + RamUsageEstimator.HumanReadableUnits(ShallowSizeOf(all)));
 
             Guard = all;
         }
 
         private long ShallowSizeOf(object[] all)
         {
-            long s = RamUsageEstimator.shallowSizeOf(all);
+            long s = RamUsageEstimator.ShallowSizeOf(all);
             foreach (object o in all)
             {
-                s += RamUsageEstimator.shallowSizeOf(o);
+                s += RamUsageEstimator.ShallowSizeOf(o);
             }
             return s;
         }
 
         private long ShallowSizeOf(object[][] all)
         {
-            long s = RamUsageEstimator.shallowSizeOf(all);
+            long s = RamUsageEstimator.ShallowSizeOf(all);
             foreach (object[] o in all)
             {
-                s += RamUsageEstimator.shallowSizeOf(o);
+                s += RamUsageEstimator.ShallowSizeOf(o);
                 foreach (object o2 in o)
                 {
-                    s += RamUsageEstimator.shallowSizeOf(o2);
+                    s += RamUsageEstimator.ShallowSizeOf(o2);
                 }
             }
             return s;
         }
 
-        [Test]
+        [Test, Timeout(180000)]
         public virtual void TestSimpleByteArrays()
         {
-            MemoryMXBean memoryMXBean = ManagementFactory.MemoryMXBean;
-
             object[][] all = new object[0][];
             try
             {
@@ -135,18 +129,18 @@ namespace Lucene.Net.Util
                 {
                     // Check the current memory consumption and provide the estimate.
                     CauseGc();
-                    MemoryUsage mu = memoryMXBean.HeapMemoryUsage;
+                    
                     long estimated = ShallowSizeOf(all);
                     if (estimated > 50 * RamUsageEstimator.ONE_MB)
                     {
                         break;
                     }
 
-                    Console.WriteLine(string.format(Locale.ROOT, "%10s\t%10s\t%10s", RamUsageEstimator.humanReadableUnits(mu.Used), RamUsageEstimator.humanReadableUnits(mu.Max), RamUsageEstimator.humanReadableUnits(estimated)));
+                    Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0}\t{1}\t{2}", RamUsageEstimator.HumanReadableUnits(GC.GetTotalMemory(false)).PadLeft(10, ' '), RamUsageEstimator.HumanReadableUnits(GC.MaxGeneration).PadLeft(10, ' '), RamUsageEstimator.HumanReadableUnits(estimated).PadLeft(10, ' ')));
 
                     // Make another batch of objects.
                     object[] seg = new object[10000];
-                    all = Arrays.copyOf(all, all.Length + 1);
+                    all = Arrays.CopyOf(all, all.Length + 1);
                     all[all.Length - 1] = seg;
                     for (int i = 0; i < seg.Length; i++)
                     {
@@ -166,23 +160,7 @@ namespace Lucene.Net.Util
         /// </summary>
         private void CauseGc()
         {
-            IList<GarbageCollectorMXBean> garbageCollectorMXBeans = ManagementFactory.GarbageCollectorMXBeans;
-            IList<long?> ccounts = new List<long?>();
-            foreach (GarbageCollectorMXBean g in garbageCollectorMXBeans)
-            {
-                ccounts.Add(g.CollectionCount);
-            }
-            IList<long?> ccounts2 = new List<long?>();
-            do
-            {
-                System.gc();
-                ccounts.Clear();
-                foreach (GarbageCollectorMXBean g in garbageCollectorMXBeans)
-                {
-                    ccounts2.Add(g.CollectionCount);
-                }
-            } while (ccounts2.Equals(ccounts));
+            GC.Collect();
         }
     }
-
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/b1203e6b/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs b/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs
index 113e730..b08739d 100644
--- a/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs
+++ b/src/Lucene.Net.Tests/core/Util/TestRamUsageEstimator.cs
@@ -1,10 +1,8 @@
-using System;
-using Lucene.Net.Util;
 using NUnit.Framework;
+using System;
 
 namespace Lucene.Net.Util
 {
-
     /*
      * Licensed to the Apache Software Foundation (ASF) under one or more
      * contributor license agreements.  See the NOTICE file distributed with
@@ -25,22 +23,22 @@ namespace Lucene.Net.Util
     [TestFixture]
     public class TestRamUsageEstimator : LuceneTestCase
     {
-        /*[Test]
+        [Test]
         public virtual void TestSanity()
         {
-            Assert.IsTrue(sizeof(new string("test string")) > ShallowSizeOfInstance(typeof(string)));
+            Assert.IsTrue(RamUsageEstimator.SizeOf("test string") > RamUsageEstimator.ShallowSizeOfInstance(typeof(string)));
 
             Holder holder = new Holder();
             holder.Holder_Renamed = new Holder("string2", 5000L);
-            Assert.IsTrue(sizeof(holder) > shallowSizeOfInstance(typeof(Holder)));
-            Assert.IsTrue(sizeof(holder) > sizeof(holder.Holder_Renamed));
+            Assert.IsTrue(RamUsageEstimator.SizeOf(holder) > RamUsageEstimator.ShallowSizeOfInstance(typeof(Holder)));
+            Assert.IsTrue(RamUsageEstimator.SizeOf(holder) > RamUsageEstimator.SizeOf(holder.Holder_Renamed));
 
-            Assert.IsTrue(shallowSizeOfInstance(typeof(HolderSubclass)) >= shallowSizeOfInstance(typeof(Holder)));
-            Assert.IsTrue(shallowSizeOfInstance(typeof(Holder)) == shallowSizeOfInstance(typeof(HolderSubclass2)));
+            Assert.IsTrue(RamUsageEstimator.ShallowSizeOfInstance(typeof(HolderSubclass)) >= RamUsageEstimator.ShallowSizeOfInstance(typeof(Holder)));
+            Assert.IsTrue(RamUsageEstimator.ShallowSizeOfInstance(typeof(Holder)) == RamUsageEstimator.ShallowSizeOfInstance(typeof(HolderSubclass2)));
 
-            string[] strings = new string[] { new string("test string"), new string("hollow"), new string("catchmaster") };
-            Assert.IsTrue(sizeof(strings) > shallowSizeOf(strings));
-        }*/
+            string[] strings = new string[] { "test string", "hollow", "catchmaster" };
+            Assert.IsTrue(RamUsageEstimator.SizeOf(strings) > RamUsageEstimator.ShallowSizeOf(strings));
+        }
 
         [Test]
         public virtual void TestStaticOverloads()
@@ -48,73 +46,74 @@ namespace Lucene.Net.Util
             Random rnd = Random();
             {
                 sbyte[] array = new sbyte[rnd.Next(1024)];
-                Assert.AreEqual(sizeof(array), sizeof((object)array));
+                assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
             }
 
             {
                 bool[] array = new bool[rnd.Next(1024)];
-                Assert.AreEqual(sizeof(array), sizeof((object)array));
+                assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
             }
 
             {
                 char[] array = new char[rnd.Next(1024)];
-                Assert.AreEqual(sizeof(array), sizeof((object)array));
+                assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
             }
 
             {
                 short[] array = new short[rnd.Next(1024)];
-                Assert.AreEqual(sizeof(array), sizeof((object)array));
+                assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
             }
 
             {
                 int[] array = new int[rnd.Next(1024)];
-                Assert.AreEqual(sizeof(array), sizeof((object)array));
+                assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
             }
 
             {
                 float[] array = new float[rnd.Next(1024)];
-                Assert.AreEqual(sizeof(array), sizeof((object)array));
+                assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
             }
 
             {
                 long[] array = new long[rnd.Next(1024)];
-                Assert.AreEqual(sizeof(array), sizeof((object)array));
+                assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
             }
 
             {
                 double[] array = new double[rnd.Next(1024)];
-                Assert.AreEqual(sizeof(array), sizeof((object)array));
+                assertEquals(RamUsageEstimator.SizeOf(array), RamUsageEstimator.SizeOf((object)array));
             }
         }
 
-        /* LUCENE PORT: The JVM is obviously not relevant to our purposes
-        [Test]
+        
+       [Test]
         public virtual void TestReferenceSize()
         {
-            if (!SupportedJVM)
-            {
-                Console.Error.WriteLine("WARN: Your JVM does not support certain Oracle/Sun extensions.");
-                Console.Error.WriteLine(" Memory estimates may be inaccurate.");
-                Console.Error.WriteLine(" Please report this to the Lucene mailing list.");
-                Console.Error.WriteLine("JVM version: " + RamUsageEstimator.JVM_INFO_STRING);
-                Console.Error.WriteLine("UnsupportedFeatures:");
-                foreach (JvmFeature f in RamUsageEstimator.UnsupportedFeatures)
-                {
-                    Console.Error.Write(" - " + f.ToString());
-                    if (f == RamUsageEstimator.JvmFeature.OBJECT_ALIGNMENT)
-                    {
-                        Console.Error.Write("; Please note: 32bit Oracle/Sun VMs don't allow exact OBJECT_ALIGNMENT retrieval, this is a known issue.");
-                    }
-                    Console.Error.WriteLine();
-                }
-            }
-
-            Assert.IsTrue(NUM_BYTES_OBJECT_REF == 4 || NUM_BYTES_OBJECT_REF == 8);
+            //LUCENENET: The JVM is obviously not relevant to our purposes
+            //if (!SupportedJVM)
+            //{
+            //    Console.Error.WriteLine("WARN: Your JVM does not support certain Oracle/Sun extensions.");
+            //    Console.Error.WriteLine(" Memory estimates may be inaccurate.");
+            //    Console.Error.WriteLine(" Please report this to the Lucene mailing list.");
+            //    Console.Error.WriteLine("JVM version: " + RamUsageEstimator.JVM_INFO_STRING);
+            //    Console.Error.WriteLine("UnsupportedFeatures:");
+            //    foreach (JvmFeature f in RamUsageEstimator.UnsupportedFeatures)
+            //    {
+            //        Console.Error.Write(" - " + f.ToString());
+            //        if (f == RamUsageEstimator.JvmFeature.OBJECT_ALIGNMENT)
+            //        {
+            //            Console.Error.Write("; Please note: 32bit Oracle/Sun VMs don't allow exact OBJECT_ALIGNMENT retrieval, this is a known issue.");
+            //        }
+            //        Console.Error.WriteLine();
+            //    }
+            //}
+
+            Assert.IsTrue(RamUsageEstimator.NUM_BYTES_OBJECT_REF == 4 || RamUsageEstimator.NUM_BYTES_OBJECT_REF == 8);
             if (!Constants.JRE_IS_64BIT)
             {
-                Assert.AreEqual("For 32bit JVMs, reference size must always be 4?", 4, NUM_BYTES_OBJECT_REF);
+                assertEquals("For 32bit JVMs, reference size must always be 4?", 4, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
             }
-        }*/
+        }
 
         private class Holder
         {