You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by mh...@apache.org on 2013/09/24 20:32:49 UTC
[13/50] [abbrv] git commit: Port: more util test classes
Port: more util test classes
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/e02cc69c
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/e02cc69c
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/e02cc69c
Branch: refs/heads/branch_4x
Commit: e02cc69c378e5673f46eb2bc344ef0cec782aa03
Parents: b713f3b
Author: James Blair <jm...@gmail.com>
Authored: Wed Jul 17 18:17:49 2013 -0400
Committer: James Blair <jm...@gmail.com>
Committed: Wed Jul 17 18:17:49 2013 -0400
----------------------------------------------------------------------
test/core/Lucene.Net.Test.csproj | 5 +-
test/core/Util/Cache/TestSimpleLRUCache.cs | 77 -----
test/core/Util/Fst/Test2BFST.cs | 335 +++++++++++++++++++
test/core/Util/Fst/TestBytesStore.cs | 408 ++++++++++++++++++++++++
test/core/Util/TestFilterIterator.cs | 5 +
test/core/Util/TestFixedBitSet.cs | 373 ++++++++++++++++++++++
test/core/Util/TestIdentityHashSet.cs | 45 +++
7 files changed, 1169 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Lucene.Net.Test.csproj
----------------------------------------------------------------------
diff --git a/test/core/Lucene.Net.Test.csproj b/test/core/Lucene.Net.Test.csproj
index 2544dd6..cccd979 100644
--- a/test/core/Lucene.Net.Test.csproj
+++ b/test/core/Lucene.Net.Test.csproj
@@ -535,10 +535,12 @@
<Compile Include="Util\Automaton\TestMinimize.cs" />
<Compile Include="Util\Automaton\TestSpecialOperations.cs" />
<Compile Include="Util\Automaton\TestUTF32ToUTF8.cs" />
- <Compile Include="Util\Cache\TestSimpleLRUCache.cs" />
<Compile Include="Util\English.cs">
<SubType>Code</SubType>
</Compile>
+ <Compile Include="Util\Fst\Test2BFST.cs" />
+ <Compile Include="Util\Fst\TestBytesStore.cs" />
+ <Compile Include="Util\Fst\TestFSTs.cs" />
<Compile Include="Util\LocalizedTestCase.cs" />
<Compile Include="Util\Paths.cs" />
<Compile Include="Util\StressRamUsageEstimator.cs" />
@@ -648,7 +650,6 @@
<Content Include="UpdatedTests.txt" />
</ItemGroup>
<ItemGroup>
- <Folder Include="Util\Fst\" />
<Folder Include="Util\Packed\" />
</ItemGroup>
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/Cache/TestSimpleLRUCache.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/Cache/TestSimpleLRUCache.cs b/test/core/Util/Cache/TestSimpleLRUCache.cs
deleted file mode 100644
index a33c2c7..0000000
--- a/test/core/Util/Cache/TestSimpleLRUCache.cs
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-using System;
-
-using NUnit.Framework;
-
-using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
-
-namespace Lucene.Net.Util.Cache
-{
-
- [TestFixture]
- public class TestSimpleLRUCache:LuceneTestCase
- {
-
- [Test]
- public virtual void TestLRUCache()
- {
- int n = 100;
- object dummy = new object();
-
- Cache<int, object> cache = new SimpleLRUCache<int, object>(n);
-
- for (int i = 0; i < n; i++)
- {
- cache.Put(i, dummy);
- }
-
- // access every 2nd item in cache
- for (int i = 0; i < n; i += 2)
- {
- Assert.IsNotNull(cache.Get(i));
- }
-
- // add n/2 elements to cache, the ones that weren't
- // touched in the previous loop should now be thrown away
- for (int i = n; i < n + (n / 2); i++)
- {
- cache.Put(i, dummy);
- }
-
- // access every 4th item in cache
- for (int i = 0; i < n; i += 4)
- {
- Assert.IsNotNull(cache.Get(i));
- }
-
- // add 3/4n elements to cache, the ones that weren't
- // touched in the previous loops should now be thrown away
- for (int i = n; i < n + (n * 3 / 4); i++)
- {
- cache.Put(i, dummy);
- }
-
- // access every 4th item in cache
- for (int i = 0; i < n; i += 4)
- {
- Assert.IsNotNull(cache.Get(i));
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/Fst/Test2BFST.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/Fst/Test2BFST.cs b/test/core/Util/Fst/Test2BFST.cs
new file mode 100644
index 0000000..857f912
--- /dev/null
+++ b/test/core/Util/Fst/Test2BFST.cs
@@ -0,0 +1,335 @@
+using System;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using Lucene.Net.Test.Support;
+using Lucene.Net.Util;
+using Lucene.Net.Util.Fst;
+using Lucene.Net.Util.Packed;
+using NUnit.Framework;
+
+namespace Lucene.Net.Test.Util.Fst
+{
+ [TestFixture]
+ [Ignore("Requires tons of heap to run (10G works)")]
+ [Timeout(360000000)] // @TimeoutSuite(millis = 100 * TimeUnits.HOUR)
+ public class Test2BFST : LuceneTestCase
+ {
+ private static long LIMIT = 3L * 1024 * 1024 * 1024;
+
+ [Test]
+ public void Test()
+ {
+ var ints = new int[7];
+ var input = new IntsRef(ints, 0, ints.Length);
+ var seed = new Random().NextLong();
+
+ Directory dir = new MMapDirectory(_TestUtil.GetTempDir("2BFST"));
+
+ for (var doPackIter = 0; doPackIter < 2; doPackIter++)
+ {
+ var doPack = doPackIter == 1;
+
+ // Build FST w/ NoOutputs and stop when nodeCount > 3B
+ if (!doPack)
+ {
+ Console.WriteLine("\nTEST: 3B nodes; doPack=false output=NO_OUTPUTS");
+ Outputs<object> outputs = NoOutputs.GetSingleton();
+ var NO_OUTPUT = outputs.GetNoOutput();
+ var b = new Builder<object>(FST.INPUT_TYPE.BYTE1, 0, 0, false, false, int.MaxValue, outputs,
+ null, doPack, PackedInts.COMPACT, true, 15);
+
+ var count = 0;
+ var r = new Random((int)seed);
+ var ints2 = new int[200];
+ var input2 = new IntsRefs(ints2, 0, ints2.Length);
+ while (true)
+ {
+ //Console.WriteLine("add: " + input + " -> " + output);
+ for (var i = 10; i < ints2.Length; i++)
+ {
+ ints2[i] = r.Next(256);
+ }
+ b.Add(input2, NO_OUTPUT);
+ count++;
+ if (count % 100000 == 0)
+ {
+ Console.WriteLine(count + ": " + b.FstSizeInBytes() + " bytes; " + b.TotStateCount + " nodes");
+ }
+ if (b.TotStateCount > LIMIT)
+ {
+ break;
+ }
+ NextInput(r, ints2);
+ }
+
+ var fst = b.Finish();
+
+ for (var verify = 0; verify < 2; verify++)
+ {
+ Console.WriteLine("\nTEST: now verify [fst size=" + fst.SizeInBytes() + "; nodeCount=" + fst.NodeCount + "; arcCount=" + fst.ArcCount + "]");
+
+ Arrays.Fill(ints2, 0);
+ r = new Random((int)seed);
+
+ for (var i = 0; i < count; i++)
+ {
+ if (i % 1000000 == 0)
+ {
+ Console.WriteLine(i + "...: ");
+ }
+ for (int j = 10; j < ints2.Length; j++)
+ {
+ ints2[j] = r.Next(256);
+ }
+ assertEquals(NO_OUTPUT, Util.get(fst, input2));
+ NextInput(r, ints2);
+ }
+
+ Console.WriteLine("\nTEST: enum all input/outputs");
+ var fstEnum = new IntsRefFSTEnum<object>(fst);
+
+ Arrays.Fill(ints2, 0);
+ r = new Random((int)seed);
+ var upto = 0;
+ while (true)
+ {
+ var pair = fstEnum.Next();
+ if (pair == null)
+ {
+ break;
+ }
+ for (int j = 10; j < ints2.Length; j++)
+ {
+ ints2[j] = r.Next(256);
+ }
+ assertEquals(input2, pair.Input);
+ assertEquals(NO_OUTPUT, pair.Output);
+ upto++;
+ NextInput(r, ints2);
+ }
+ assertEquals(count, upto);
+
+ if (verify == 0)
+ {
+ Console.WriteLine("\nTEST: save/load FST and re-verify");
+ var output = dir.CreateOutput("fst", IOContext.DEFAULT);
+ fst.Save(output);
+ output.Dispose();
+ var input3 = dir.OpenInput("fst", IOContext.DEFAULT);
+ fst = new FST<object>(input3, outputs);
+ input3.Dispose();
+ }
+ else
+ {
+ dir.DeleteFile("fst");
+ }
+ }
+ }
+
+ // Build FST w/ ByteSequenceOutputs and stop when FST
+ // size = 3GB
+ {
+ Console.WriteLine("\nTEST: 3 GB size; doPack=" + doPack + " outputs=bytes");
+ Outputs<BytesRef> outputs = ByteSequenceOutputs.GetSingleton();
+ var b = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs,
+ null, doPack, PackedInts.COMPACT, true, 15);
+
+ var outputBytes = new byte[20];
+ var output = new BytesRef(outputBytes);
+ Arrays.Fill(ints, 0);
+ var count = 0;
+ var r = new Random(seed);
+ while (true)
+ {
+ r.NextBytes(outputBytes);
+ //Console.WriteLine("add: " + input + " -> " + output);
+ b.Add(input, BytesRef.DeepCopyOf(output));
+ count++;
+ if (count % 1000000 == 0)
+ {
+ Console.WriteLine(count + "...: " + b.FstSizeInBytes() + " bytes");
+ }
+ if (b.FstSizeInBytes() > LIMIT)
+ {
+ break;
+ }
+ NextInput(r, ints);
+ }
+
+ FST<BytesRef> fst = b.Finish();
+ for (int verify = 0; verify < 2; verify++)
+ {
+
+ Console.WriteLine("\nTEST: now verify [fst size=" + fst.SizeInBytes() + "; nodeCount=" + fst.NodeCount + "; arcCount=" + fst.ArcCount + "]");
+
+ r = new Random(seed);
+ Arrays.Fill(ints, 0);
+
+ for (int i = 0; i < count; i++)
+ {
+ if (i % 1000000 == 0)
+ {
+ Console.WriteLine(i + "...: ");
+ }
+ r.NextBytes(outputBytes);
+ assertEquals(output, Util.get(fst, input));
+ NextInput(r, ints);
+ }
+
+ Console.WriteLine("\nTEST: enum all input/outputs");
+ var fstEnum = new IntsRefFSTEnum<BytesRef>(fst);
+
+ Arrays.Fill(ints, 0);
+ r = new Random((int)seed);
+ int upto = 0;
+ while (true)
+ {
+ var pair = fstEnum.Next();
+ if (pair == null)
+ {
+ break;
+ }
+ assertEquals(input, pair.Input);
+ r.NextBytes(outputBytes);
+ assertEquals(output, pair.Output);
+ upto++;
+ NextInput(r, ints);
+ }
+ assertEquals(count, upto);
+
+ if (verify == 0)
+ {
+ Console.WriteLine("\nTEST: save/load FST and re-verify");
+ var output2 = dir.CreateOutput("fst", IOContext.DEFAULT);
+ fst.Save(output2);
+ output2.Dispose();
+ var input4 = dir.OpenInput("fst", IOContext.DEFAULT);
+ fst = new FST<BytesRef>(input4, outputs);
+ input4.Dispose();
+ }
+ else
+ {
+ dir.DeleteFile("fst");
+ }
+ }
+ }
+
+ // Build FST w/ PositiveIntOutputs and stop when FST
+ // size = 3GB
+ {
+ Console.WriteLine("\nTEST: 3 GB size; doPack=" + doPack + " outputs=long");
+ Outputs<long> outputs = PositiveIntOutputs.GetSingleton();
+ Builder<long> b = new Builder<long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, int.MaxValue, outputs,
+ null, doPack, PackedInts.COMPACT, true, 15);
+
+ long output = 1;
+
+ Arrays.Fill(ints, 0);
+ var count = 0;
+ var r = new Random(seed);
+ while (true)
+ {
+ //Console.WriteLine("add: " + input + " -> " + output);
+ b.Add(input, output);
+ output += 1 + r.Next(10);
+ count++;
+ if (count % 1000000 == 0)
+ {
+ Console.WriteLine(count + "...: " + b.FstSizeInBytes() + " bytes");
+ }
+ if (b.FstSizeInBytes() > LIMIT)
+ {
+ break;
+ }
+ NextInput(r, ints);
+ }
+
+ FST<long> fst = b.Finish();
+
+ for (int verify = 0; verify < 2; verify++)
+ {
+
+ Console.WriteLine("\nTEST: now verify [fst size=" + fst.SizeInBytes() + "; nodeCount=" + fst.NodeCount + "; arcCount=" + fst.ArcCount + "]");
+
+ Arrays.Fill(ints, 0);
+
+ output = 1;
+ r = new Random(seed);
+ for (int i = 0; i < count; i++)
+ {
+ if (i % 1000000 == 0)
+ {
+ Console.WriteLine(i + "...: ");
+ }
+
+ // forward lookup:
+ assertEquals(output, Util.get(fst, input).longValue());
+ // reverse lookup:
+ assertEquals(input, Util.getByOutput(fst, output));
+ output += 1 + r.Next(10);
+ NextInput(r, ints);
+ }
+
+ Console.WriteLine("\nTEST: enum all input/outputs");
+ IntsRefFSTEnum<long> fstEnum = new IntsRefFSTEnum<long>(fst);
+
+ Arrays.Fill(ints, 0);
+ r = new Random((int)seed);
+ int upto = 0;
+ output = 1;
+ while (true)
+ {
+ var pair = fstEnum.Next();
+ if (pair == null)
+ {
+ break;
+ }
+ assertEquals(input, pair.Input);
+ assertEquals(output, pair.Output);
+ output += 1 + r.Next(10);
+ upto++;
+ NextInput(r, ints);
+ }
+ assertEquals(count, upto);
+
+ if (verify == 0)
+ {
+ Console.WriteLine("\nTEST: save/load FST and re-verify");
+ var output3 = dir.CreateOutput("fst", IOContext.DEFAULT);
+ fst.Save(output3);
+ output3.Dispose();
+ var input5 = dir.OpenInput("fst", IOContext.DEFAULT);
+ fst = new FST<long>(input5, outputs);
+ input5.Dispose();
+ }
+ else
+ {
+ dir.DeleteFile("fst");
+ }
+ }
+ }
+ }
+ dir.Dispose();
+ }
+
+ private void NextInput(Random r, int[] ints)
+ {
+ var downTo = 6;
+ while (downTo >= 0)
+ {
+ // Must add random amounts (and not just 1) because
+ // otherwise FST outsmarts us and remains tiny:
+ ints[downTo] += 1 + r.Next(10);
+ if (ints[downTo] < 256)
+ {
+ break;
+ }
+ else
+ {
+ ints[downTo] = 0;
+ downTo--;
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/Fst/TestBytesStore.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/Fst/TestBytesStore.cs b/test/core/Util/Fst/TestBytesStore.cs
new file mode 100644
index 0000000..df435a2
--- /dev/null
+++ b/test/core/Util/Fst/TestBytesStore.cs
@@ -0,0 +1,408 @@
+using System;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using Lucene.Net.Test.Support;
+using Lucene.Net.Util;
+using Lucene.Net.Util.Fst;
+using NUnit.Framework;
+
+namespace Lucene.Net.Test.Util.Fst
+{
+ [TestFixture]
+ public class TestBytesStore : LuceneTestCase
+ {
+ [Test]
+ public void TestRandom()
+ {
+
+ int iters = AtLeast(10);
+ for (var iter = 0; iter < iters; iter++)
+ {
+ int numBytes = _TestUtil.NextInt(new Random(), 1, 200000);
+ var expected = new sbyte[numBytes];
+ int blockBits = _TestUtil.NextInt(new Random(), 8, 15);
+ var bytes = new BytesStore(blockBits);
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: iter=" + iter + " numBytes=" + numBytes + " blockBits=" + blockBits);
+ }
+
+ var pos = 0;
+ while (pos < numBytes)
+ {
+ int op = new Random().Next(8);
+ if (VERBOSE)
+ {
+ Console.WriteLine(" cycle pos=" + pos);
+ }
+ switch (op)
+ {
+
+ case 0:
+ {
+ // write random byte
+ var b = (sbyte)new Random().Next(256);
+ if (VERBOSE)
+ {
+ Console.WriteLine(" writeByte b=" + b);
+ }
+
+ expected[pos++] = b;
+ bytes.WriteByte(b);
+ }
+ break;
+
+ case 1:
+ {
+ // write random byte[]
+ var len = new Random().Next(Math.Min(numBytes - pos, 100));
+ var temp = new byte[len];
+ new Random().NextBytes(temp);
+ if (VERBOSE)
+ {
+ Console.WriteLine(" writeBytes len=" + len + " bytes=" + Arrays.ToString(temp));
+ }
+ Array.Copy(temp, 0, expected, pos, temp.Length);
+ bytes.WriteBytes(temp, 0, temp.Length);
+ pos += len;
+ }
+ break;
+
+ case 2:
+ {
+ // write int @ absolute pos
+ if (pos > 4)
+ {
+ int x = new Random().Next();
+ int randomPos = new Random().Next(pos - 4);
+ if (VERBOSE)
+ {
+ Console.WriteLine(" abs writeInt pos=" + randomPos + " x=" + x);
+ }
+ bytes.WriteInt(randomPos, x);
+ expected[randomPos++] = (sbyte)(x >> 24);
+ expected[randomPos++] = (sbyte)(x >> 16);
+ expected[randomPos++] = (sbyte)(x >> 8);
+ expected[randomPos++] = (sbyte)x;
+ }
+ }
+ break;
+
+ case 3:
+ {
+ // reverse bytes
+ if (pos > 1)
+ {
+ int len = _TestUtil.NextInt(new Random(), 2, Math.Min(100, pos));
+ int start;
+ if (len == pos)
+ {
+ start = 0;
+ }
+ else
+ {
+ start = new Random().Next(pos - len);
+ }
+ var end = start + len - 1;
+ if (VERBOSE)
+ {
+ Console.WriteLine(" reverse start=" + start + " end=" + end + " len=" + len + " pos=" + pos);
+ }
+ bytes.Reverse(start, end);
+
+ while (start <= end)
+ {
+ var b = expected[end];
+ expected[end] = expected[start];
+ expected[start] = b;
+ start++;
+ end--;
+ }
+ }
+ }
+ break;
+
+ case 4:
+ {
+ // abs write random byte[]
+ if (pos > 2)
+ {
+ int randomPos = new Random().Next(pos - 1);
+ int len = _TestUtil.NextInt(new Random(), 1, Math.Min(pos - randomPos - 1, 100));
+ byte[] temp = new byte[len];
+ new Random().NextBytes(temp);
+ if (VERBOSE)
+ {
+ Console.WriteLine(" abs writeBytes pos=" + randomPos + " len=" + len + " bytes=" + Arrays.ToString(temp));
+ }
+ Array.Copy(temp, 0, expected, randomPos, temp.Length);
+ bytes.WriteBytes(randomPos, temp, 0, temp.Length);
+ }
+ }
+ break;
+
+ case 5:
+ {
+ // copyBytes
+ if (pos > 1)
+ {
+ int src = new Random().Next(pos - 1);
+ int dest = _TestUtil.NextInt(new Random(), src + 1, pos - 1);
+ int len = _TestUtil.NextInt(new Random(), 1, Math.Min(300, pos - dest));
+ if (VERBOSE)
+ {
+ Console.WriteLine(" copyBytes src=" + src + " dest=" + dest + " len=" + len);
+ }
+ Array.Copy(expected, src, expected, dest, len);
+ bytes.CopyBytes(src, dest, len);
+ }
+ }
+ break;
+
+ case 6:
+ {
+ // skip
+ var len = new Random().Next(Math.Min(100, numBytes - pos));
+
+ if (VERBOSE)
+ {
+ Console.WriteLine(" skip len=" + len);
+ }
+
+ pos += len;
+ bytes.SkipBytes(len);
+
+ // NOTE: must fill in zeros in case truncate was
+ // used, else we get false fails:
+ if (len > 0)
+ {
+ var zeros = new sbyte[len];
+ bytes.WriteBytes(pos - len, zeros, 0, len);
+ }
+ }
+ break;
+
+ case 7:
+ {
+ // absWriteByte
+ if (pos > 0)
+ {
+ var dest = new Random().Next(pos);
+ var b = (sbyte)new Random().Next(256);
+ expected[dest] = b;
+ bytes.WriteByte(dest, b);
+ }
+ break;
+ }
+ }
+
+ Assert.AreEqual(pos, bytes.GetPosition());
+
+ if (pos > 0 && new Random().Next(50) == 17)
+ {
+ // truncate
+ int len = _TestUtil.NextInt(new Random(), 1, Math.Min(pos, 100));
+ bytes.Truncate(pos - len);
+ pos -= len;
+ Arrays.Fill(expected, pos, pos + len, (sbyte)0);
+ if (VERBOSE)
+ {
+ Console.WriteLine(" truncate len=" + len + " newPos=" + pos);
+ }
+ }
+
+ if ((pos > 0 && new Random().Next(200) == 17))
+ {
+ Verify(bytes, expected, pos);
+ }
+ }
+
+ BytesStore bytesToVerify;
+
+ if (new Random().NextBool())
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: save/load bytes");
+ }
+ Directory dir = NewDirectory();
+ var output = dir.CreateOutput("bytes", IOContext.DEFAULT);
+ bytes.WriteTo(output);
+ output.Dispose();
+ var input = dir.OpenInput("bytes", IOContext.DEFAULT);
+ bytesToVerify = new BytesStore(input, numBytes, _TestUtil.NextInt(new Random(), 256, int.MaxValue));
+ input.Dispose();
+ dir.Dispose();
+ }
+ else
+ {
+ bytesToVerify = bytes;
+ }
+
+ Verify(bytesToVerify, expected, numBytes);
+ }
+ }
+
+ private void Verify(BytesStore bytes, sbyte[] expected, int totalLength)
+ {
+ Assert.AreEqual(totalLength, bytes.GetPosition());
+ if (totalLength == 0)
+ {
+ return;
+ }
+ if (VERBOSE)
+ {
+ Console.WriteLine(" verify...");
+ }
+
+ // First verify whole thing in one blast:
+ var actual = new sbyte[totalLength];
+ if (new Random().NextBool())
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine(" bulk: reversed");
+ }
+ // reversed
+ var reverseReader = bytes.GetReverseReader();
+ Assert.IsTrue(reverseReader.Reversed());
+ reverseReader.Position = totalLength - 1;
+ reverseReader.ReadBytes(actual, 0, actual.Length);
+ var start = 0;
+ var end = totalLength - 1;
+ while (start < end)
+ {
+ var b = actual[start];
+ actual[start] = actual[end];
+ actual[end] = b;
+ start++;
+ end--;
+ }
+ }
+ else
+ {
+ // forward
+ if (VERBOSE)
+ {
+ Console.WriteLine(" bulk: forward");
+ }
+ var forwardReader = bytes.GetForwardReader();
+ Assert.IsFalse(forwardReader.Reversed());
+ forwardReader.ReadBytes(actual, 0, actual.Length);
+ }
+
+ for (int i = 0; i < totalLength; i++)
+ {
+ Assert.AreEqual(expected[i], actual[i], "byte @ index=" + i);
+ }
+
+ FST.BytesReader r;
+
+ // Then verify ops:
+ bool reversed = new Random().NextBool();
+ if (reversed)
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine(" ops: reversed");
+ }
+ r = bytes.GetReverseReader();
+ }
+ else
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine(" ops: forward");
+ }
+ r = bytes.GetForwardReader();
+ }
+
+ if (totalLength > 1)
+ {
+ int numOps = _TestUtil.NextInt(new Random(), 100, 200);
+ for (int op = 0; op < numOps; op++)
+ {
+
+ int numBytes = new Random().Next(Math.Min(1000, totalLength - 1));
+ int pos;
+ if (reversed)
+ {
+ pos = _TestUtil.NextInt(new Random(), numBytes, totalLength - 1);
+ }
+ else
+ {
+ pos = new Random().Next(totalLength - numBytes);
+ }
+ if (VERBOSE)
+ {
+ Console.WriteLine(" op iter=" + op + " reversed=" + reversed + " numBytes=" + numBytes + " pos=" + pos);
+ }
+ var temp = new sbyte[numBytes];
+ r.Position = pos;
+ Assert.AreEqual(pos, r.Position);
+ r.ReadBytes(temp, 0, temp.Length);
+ for (int i = 0; i < numBytes; i++)
+ {
+ sbyte expectedByte;
+ if (reversed)
+ {
+ expectedByte = expected[pos - i];
+ }
+ else
+ {
+ expectedByte = expected[pos + i];
+ }
+ Assert.AreEqual(expectedByte, temp[i], "byte @ index=" + i);
+ }
+
+ int left;
+ int expectedPos;
+
+ if (reversed)
+ {
+ expectedPos = pos - numBytes;
+ left = (int)r.Position;
+ }
+ else
+ {
+ expectedPos = pos + numBytes;
+ left = (int)(totalLength - r.Position);
+ }
+ Assert.AreEqual(expectedPos, r.Position);
+
+ if (left > 4)
+ {
+ int skipBytes = new Random().Next(left - 4);
+
+ int expectedInt = 0;
+ if (reversed)
+ {
+ expectedPos -= skipBytes;
+ expectedInt |= (expected[expectedPos--] & 0xFF) << 24;
+ expectedInt |= (expected[expectedPos--] & 0xFF) << 16;
+ expectedInt |= (expected[expectedPos--] & 0xFF) << 8;
+ expectedInt |= (expected[expectedPos--] & 0xFF);
+ }
+ else
+ {
+ expectedPos += skipBytes;
+ expectedInt |= (expected[expectedPos++] & 0xFF) << 24;
+ expectedInt |= (expected[expectedPos++] & 0xFF) << 16;
+ expectedInt |= (expected[expectedPos++] & 0xFF) << 8;
+ expectedInt |= (expected[expectedPos++] & 0xFF);
+ }
+
+ if (VERBOSE)
+ {
+ Console.WriteLine(" skip numBytes=" + skipBytes);
+ Console.WriteLine(" readInt");
+ }
+
+ r.SkipBytes(skipBytes);
+ Assert.AreEqual(expectedInt, r.ReadInt());
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/TestFilterIterator.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/TestFilterIterator.cs b/test/core/Util/TestFilterIterator.cs
new file mode 100644
index 0000000..04816d4
--- /dev/null
+++ b/test/core/Util/TestFilterIterator.cs
@@ -0,0 +1,5 @@
+namespace Lucene.Net.Test.Util
+{
+ // There is no FilterIterator class, as LINQ handles this very well with the Where method.
+ // This file is here to help with matching up files with the java file structure.
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/TestFixedBitSet.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/TestFixedBitSet.cs b/test/core/Util/TestFixedBitSet.cs
new file mode 100644
index 0000000..82d3e3c
--- /dev/null
+++ b/test/core/Util/TestFixedBitSet.cs
@@ -0,0 +1,373 @@
+using System;
+using System.Collections;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using Lucene.Net.Test.Support;
+using Lucene.Net.Util;
+using NUnit.Framework;
+
+namespace Lucene.Net.Test.Util
+{
+ [TestFixture]
+ public class TestFixedBitSet : LuceneTestCase
+ {
+ internal virtual void DoGet(BitArray a, FixedBitSet b)
+ {
+ int max = b.Length;
+ for (var i = 0; i < max; i++)
+ {
+ if (a[i] != b[i])
+ {
+ Fail("mismatch: BitArray=[" + i + "]=" + a[i]);
+ }
+ }
+ }
+
+ internal virtual void doNextSetBit(BitArray a, FixedBitSet b)
+ {
+ int aa = -1, bb = -1;
+ do
+ {
+ aa = a.NextSetBit(aa + 1);
+ bb = bb < b.Length - 1 ? b.NextSetBit(bb + 1) : -1;
+ Assert.Equals(aa, bb);
+ } while (aa >= 0);
+ }
+
+ internal virtual void doPrevSetBit(BitArray a, FixedBitSet b)
+ {
+ int aa = a.Length + new Random().Next(100);
+ int bb = aa;
+ do
+ {
+ // aa = a.prevSetBit(aa-1);
+ aa--;
+ while ((aa >= 0) && (!a[aa]))
+ {
+ aa--;
+ }
+ if (b.Length == 0)
+ {
+ bb = -1;
+ }
+ else if (bb > b.Length - 1)
+ {
+ bb = b.PrevSetBit(b.Length - 1);
+ }
+ else if (bb < 1)
+ {
+ bb = -1;
+ }
+ else
+ {
+ bb = bb >= 1 ? b.PrevSetBit(bb - 1) : -1;
+ }
+ Assert.Equals(aa, bb);
+ } while (aa >= 0);
+ }
+
+ // test interleaving different FixedBitSetIterator.next()/skipTo()
+ internal virtual void DoIterate(BitArray a, FixedBitSet b, int mode)
+ {
+ if (mode == 1) DoIterate1(a, b);
+ if (mode == 2) DoIterate2(a, b);
+ }
+
+ internal virtual void DoIterate1(BitArray a, FixedBitSet b)
+ {
+ int aa = -1, bb = -1;
+ var iterator = b.Iterator();
+ do
+ {
+ aa = a.NextSetBit(aa + 1);
+ bb = (bb < b.Length && new Random().NextBool()) ? iterator.NextDoc() : iterator.Advance(bb + 1);
+ Assert.Equals(aa == -1 ? DocIdSetIterator.NO_MORE_DOCS : aa, bb);
+ } while (aa >= 0);
+ }
+
+ internal virtual void DoIterate2(BitArray a, FixedBitSet b)
+ {
+ int aa = -1, bb = -1;
+ var iterator = b.Iterator();
+ do
+ {
+ aa = a.NextSetBit(aa + 1);
+ bb = new Random().NextBool() ? iterator.NextDoc() : iterator.Advance(bb + 1);
+ Assert.Equals(aa == -1 ? DocIdSetIterator.NO_MORE_DOCS : aa, bb);
+ } while (aa >= 0);
+ }
+
+ internal virtual void DoRandomSets(int maxSize, int iter, int mode)
+ {
+ BitArray a0 = null;
+ FixedBitSet b0 = null;
+
+ var random = new Random();
+
+ for (var i = 0; i < iter; i++)
+ {
+ int sz = _TestUtil.Next(random, 2, maxSize);
+ var a = new BitArray(sz);
+ var b = new FixedBitSet(sz);
+
+ // test the various ways of setting bits
+ if (sz > 0)
+ {
+ var nOper = random.Next(sz);
+ for (var j = 0; j < nOper; j++)
+ {
+ int idx;
+
+ idx = random.Next(sz);
+ a.Set(idx);
+ b.Set(idx);
+
+ idx = random.Next(sz);
+ a.Clear(idx);
+ b.Clear(idx);
+
+ idx = random.Next(sz);
+ a.Flip(idx);
+ b.Flip(idx, idx + 1);
+
+ idx = random.Next(sz);
+ a.Flip(idx);
+ b.Flip(idx, idx + 1);
+ bool val2 = b[idx];
+ bool val = b.GetAndSet(idx);
+ Assert.IsTrue(val2 == val);
+ Assert.IsTrue(b[idx]);
+
+ if (!val) b.Clear(idx);
+ Assert.IsTrue(b[idx] == val);
+ }
+ }
+
+ // test that the various ways of accessing the bits are equivalent
+ DoGet(a, b);
+
+ // test ranges, including possible extension
+ int fromIndex, toIndex;
+ fromIndex = random.Next(sz / 2);
+ toIndex = fromIndex + random.Next(sz - fromIndex);
+ var aa = (BitArray)a.Clone(); aa.Flip(fromIndex, toIndex);
+ var bb = b.Clone(); bb.Flip(fromIndex, toIndex);
+
+ DoIterate(aa, bb, mode); // a problem here is from Flip or DoIterate
+
+ fromIndex = random.Next(sz / 2);
+ toIndex = fromIndex + random.Next(sz - fromIndex);
+ aa = (BitArray)a.Clone(); aa.Clear(fromIndex, toIndex);
+ bb = b.Clone(); bb.Clear(fromIndex, toIndex);
+
+ doNextSetBit(aa, bb); // a problem here is from Clear() or nextSetBit
+
+ doPrevSetBit(aa, bb);
+
+ fromIndex = random.Next(sz / 2);
+ toIndex = fromIndex + random.Next(sz - fromIndex);
+ aa = (BitArray)a.Clone(); aa.Set(fromIndex, toIndex);
+ bb = b.Clone(); bb.Set(fromIndex, toIndex);
+
+ doNextSetBit(aa, bb); // a problem here is from set() or nextSetBit
+
+ doPrevSetBit(aa, bb);
+
+ if (b0 != null && b0.Length <= b.Length)
+ {
+ Assert.Equals(a.Cardinality(), b.Cardinality());
+
+ var a_and = (BitArray)a.Clone(); a_and.And(a0);
+ var a_or = (BitArray)a.Clone(); a_or.Or(a0);
+ var a_andn = (BitArray)a.Clone(); a_andn.AndNot(a0);
+
+ var b_and = b.Clone(); Assert.Equals(b, b_and); b_and.And(b0);
+ var b_or = b.Clone(); b_or.Or(b0);
+ var b_andn = b.Clone(); b_andn.AndNot(b0);
+
+ Assert.Equals(a0.Cardinality(), b0.Cardinality());
+ Assert.Equals(a_or.Cardinality(), b_or.Cardinality());
+
+ DoIterate(a_and, b_and, mode);
+ DoIterate(a_or, b_or, mode);
+ DoIterate(a_andn, b_andn, mode);
+
+ Assert.Equals(a_and.Cardinality(), b_and.Cardinality());
+ Assert.Equals(a_or.Cardinality(), b_or.Cardinality());
+ Assert.Equals(a_andn.Cardinality(), b_andn.Cardinality());
+ }
+
+ a0 = a;
+ b0 = b;
+ }
+ }
+
+ // large enough to flush obvious bugs, small enough to run in <.5 sec as part of a
+ // larger testsuite.
+ [Test]
+ public void TestSmall()
+ {
+ DoRandomSets(AtLeast(1200), AtLeast(1000), 1);
+ DoRandomSets(AtLeast(1200), AtLeast(1000), 2);
+ }
+
+ // uncomment to run a bigger test (~2 minutes).
+ /*
+ [Test]
+ public void TestBig() {
+ DoRandomSets(2000,200000, 1);
+ DoRandomSets(2000,200000, 2);
+ }
+ */
+
+ [Test]
+ public void TestEquals()
+ {
+ var random = new Random();
+
+ // This test can't handle numBits==0:
+ var numBits = random.Next(2000) + 1;
+ var b1 = new FixedBitSet(numBits);
+ var b2 = new FixedBitSet(numBits);
+ Assert.IsTrue(b1.Equals(b2));
+ Assert.IsTrue(b2.Equals(b1));
+ for (var iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
+ {
+ var idx = random.Next(numBits);
+ if (!b1[idx])
+ {
+ b1.Set(idx);
+ Assert.IsFalse(b1.Equals(b2));
+ Assert.IsFalse(b2.Equals(b1));
+ b2.Set(idx);
+ Assert.IsTrue(b1.Equals(b2));
+ Assert.IsTrue(b2.Equals(b1));
+ }
+ }
+
+ // try different type of object
+ Assert.IsFalse(b1.Equals(new Object()));
+ }
+
+ [Test]
+ public void TestHashCodeEquals()
+ {
+ var random = new Random();
+
+ // This test can't handle numBits==0:
+ var numBits = random.Next(2000) + 1;
+ var b1 = new FixedBitSet(numBits);
+ var b2 = new FixedBitSet(numBits);
+ Assert.IsTrue(b1.Equals(b2));
+ Assert.IsTrue(b2.Equals(b1));
+ for (var iter = 0; iter < 10 * RANDOM_MULTIPLIER; iter++)
+ {
+ int idx = random.Next(numBits);
+ if (!b1[idx])
+ {
+ b1.Set(idx);
+ Assert.IsFalse(b1.Equals(b2));
+ Assert.IsFalse(b1.GetHashCode() == b2.GetHashCode());
+ b2.Set(idx);
+ Assert.Equals(b1, b2);
+ Assert.Equals(b1.GetHashCode(), b2.GetHashCode());
+ }
+ }
+ }
+
+ [Test]
+ public void TestSmallBitSets()
+ {
+ // Make sure size 0-10 bit sets are OK:
+ for (var numBits = 0; numBits < 10; numBits++)
+ {
+ var b1 = new FixedBitSet(numBits);
+ var b2 = new FixedBitSet(numBits);
+ Assert.IsTrue(b1.Equals(b2));
+ Assert.Equals(b1.GetHashCode(), b2.GetHashCode());
+ Assert.Equals(0, b1.Cardinality());
+ if (numBits > 0)
+ {
+ b1.Set(0, numBits);
+ Assert.Equals(numBits, b1.Cardinality());
+ b1.Flip(0, numBits);
+ Assert.Equals(0, b1.Cardinality());
+ }
+ }
+ }
+
+ private FixedBitSet MakeFixedBitSet(int[] a, int numBits)
+ {
+ var random = new Random();
+
+ FixedBitSet bs;
+ if (random.NextBool())
+ {
+ var bits2words = FixedBitSet.Bits2Words(numBits);
+ var words = new long[bits2words + random.Next(100)];
+ for (var i = bits2words; i < words.Length; i++)
+ {
+ words[i] = random.NextLong();
+ }
+ bs = new FixedBitSet(words, numBits);
+
+ }
+ else
+ {
+ bs = new FixedBitSet(numBits);
+ }
+ foreach (var e in a)
+ {
+ bs.Set(e);
+ }
+ return bs;
+ }
+
+ private BitArray MakeBitSet(int[] a)
+ {
+ var bs = new BitArray();
+ foreach (var e in a)
+ {
+ bs.Set(e);
+ }
+ return bs;
+ }
+
+ private void CheckPrevSetBitArray(int[] a, int numBits)
+ {
+ var obs = MakeFixedBitSet(a, numBits);
+ var bs = MakeBitSet(a);
+ doPrevSetBit(bs, obs);
+ }
+
+ [Test]
+ public void TestPrevSetBit()
+ {
+ CheckPrevSetBitArray(new int[] { }, 0);
+ CheckPrevSetBitArray(new int[] { 0 }, 1);
+ CheckPrevSetBitArray(new int[] { 0, 2 }, 3);
+ }
+
+ private void CheckNextSetBitArray(int[] a, int numBits)
+ {
+ var obs = MakeFixedBitSet(a, numBits);
+ var bs = MakeBitSet(a);
+ doNextSetBit(bs, obs);
+ }
+
+ [Test]
+ public void TestNextBitSet()
+ {
+ var random = new Random();
+
+ var setBits = new int[0 + random.Next(1000)];
+ for (var i = 0; i < setBits.Length; i++)
+ {
+ setBits[i] = random.Next(setBits.Length);
+ }
+ CheckNextSetBitArray(setBits, setBits.Length + random.Next(10));
+
+ CheckNextSetBitArray(new int[0], setBits.Length + random.Next(10));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/e02cc69c/test/core/Util/TestIdentityHashSet.cs
----------------------------------------------------------------------
diff --git a/test/core/Util/TestIdentityHashSet.cs b/test/core/Util/TestIdentityHashSet.cs
new file mode 100644
index 0000000..0c7e1e7
--- /dev/null
+++ b/test/core/Util/TestIdentityHashSet.cs
@@ -0,0 +1,45 @@
+using System;
+using System.Collections.Generic;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using NUnit.Framework;
+
+namespace Lucene.Net.Test.Util
+{
+ [TestFixture]
+ public class TestIdentityHashSet : LuceneTestCase
+ {
+ public void testCheck()
+ {
+ var rnd = new Random();
+ ISet<object> jdk = Collections.NewSetFromMap(
+ new IdentityHashMap<object, bool>());
+ RamUsageEstimator.IdentityHashSet<object> us = new RamUsageEstimator.IdentityHashSet<object>();
+
+ var max = 100000;
+ var threshold = 256;
+ for (var i = 0; i < max; i++)
+ {
+ // some of these will be interned and some will not so there will be collisions.
+ var v = rnd.Next(threshold);
+
+ bool e1 = jdk.Contains(v);
+ bool e2 = us.Contains(v);
+ Assert.Equals(e1, e2);
+
+ e1 = jdk.Add(v);
+ e2 = us.Add(v);
+ Assert.Equals(e1, e2);
+ }
+
+ ISet<object> collected = Collections.NewSetFromMap(
+ new IdentityHashMap<object, bool>());
+ foreach (var o in us)
+ {
+ collected.Add(o);
+ }
+
+ Assert.Equals(collected, jdk);
+ }
+ }
+}