You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/02/26 23:37:21 UTC
[33/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests:
Removed \core directory and put its contents in root directory
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestFlushByRamOrCountsPolicy.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestFlushByRamOrCountsPolicy.cs b/src/Lucene.Net.Tests/Index/TestFlushByRamOrCountsPolicy.cs
new file mode 100644
index 0000000..001f9e9
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestFlushByRamOrCountsPolicy.cs
@@ -0,0 +1,477 @@
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Index
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using Document = Lucene.Net.Documents.Document;
+ using ThreadState = Lucene.Net.Index.DocumentsWriterPerThreadPool.ThreadState;
+ using Directory = Lucene.Net.Store.Directory;
+ using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+ using LineFileDocs = Lucene.Net.Util.LineFileDocs;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+ using Lucene.Net.Support;
+ using NUnit.Framework;
+
+ [TestFixture]
+ public class TestFlushByRamOrCountsPolicy : LuceneTestCase
+ {
+
+ private static LineFileDocs LineDocFile;
+
+ [OneTimeSetUp]
+ public static void BeforeClass()
+ {
+ LineDocFile = new LineFileDocs(Random(), DefaultCodecSupportsDocValues());
+ }
+
+ [OneTimeTearDown]
+ public static void AfterClass()
+ {
+ LineDocFile.Dispose();
+ LineDocFile = null;
+ }
+
+ [Test]
+ public virtual void TestFlushByRam()
+ {
+ double ramBuffer = (TEST_NIGHTLY ? 1 : 10) + AtLeast(2) + Random().NextDouble();
+ RunFlushByRam(1 + Random().Next(TEST_NIGHTLY ? 5 : 1), ramBuffer, false);
+ }
+
+ [Test]
+ public virtual void TestFlushByRamLargeBuffer()
+ {
+ // with a 256 mb ram buffer we should never stall
+ RunFlushByRam(1 + Random().Next(TEST_NIGHTLY ? 5 : 1), 256d, true);
+ }
+
+ protected internal virtual void RunFlushByRam(int numThreads, double maxRamMB, bool ensureNotStalled)
+ {
+ int numDocumentsToIndex = 10 + AtLeast(30);
+ AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
+ Directory dir = NewDirectory();
+ MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
+ MockAnalyzer analyzer = new MockAnalyzer(Random());
+ analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
+
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetFlushPolicy(flushPolicy);
+ int numDWPT = 1 + AtLeast(2);
+ DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
+ iwc.SetIndexerThreadPool(threadPool);
+ iwc.SetRAMBufferSizeMB(maxRamMB);
+ iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
+ Assert.IsFalse(flushPolicy.FlushOnDocCount);
+ Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
+ Assert.IsTrue(flushPolicy.FlushOnRAM);
+ DocumentsWriter docsWriter = writer.DocsWriter;
+ Assert.IsNotNull(docsWriter);
+ DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
+ Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
+
+ IndexThread[] threads = new IndexThread[numThreads];
+ for (int x = 0; x < threads.Length; x++)
+ {
+ threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, false);
+ threads[x].Start();
+ }
+
+ for (int x = 0; x < threads.Length; x++)
+ {
+ threads[x].Join();
+ }
+ long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
+ Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads);
+ Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
+ Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
+ Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
+ AssertActiveBytesAfter(flushControl);
+ if (flushPolicy.HasMarkedPending)
+ {
+ Assert.IsTrue(maxRAMBytes < flushControl.peakActiveBytes);
+ }
+ if (ensureNotStalled)
+ {
+ Assert.IsFalse(docsWriter.flushControl.stallControl.WasStalled);
+ }
+ writer.Dispose();
+ Assert.AreEqual(0, flushControl.ActiveBytes);
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestFlushDocCount()
+ {
+ int[] numThreads = new int[] { 2 + AtLeast(1), 1 };
+ for (int i = 0; i < numThreads.Length; i++)
+ {
+
+ int numDocumentsToIndex = 50 + AtLeast(30);
+ AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
+ Directory dir = NewDirectory();
+ MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetFlushPolicy(flushPolicy);
+
+ int numDWPT = 1 + AtLeast(2);
+ DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
+ iwc.SetIndexerThreadPool(threadPool);
+ iwc.SetMaxBufferedDocs(2 + AtLeast(10));
+ iwc.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
+ Assert.IsTrue(flushPolicy.FlushOnDocCount);
+ Assert.IsFalse(flushPolicy.FlushOnDeleteTerms);
+ Assert.IsFalse(flushPolicy.FlushOnRAM);
+ DocumentsWriter docsWriter = writer.DocsWriter;
+ Assert.IsNotNull(docsWriter);
+ DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
+ Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
+
+ IndexThread[] threads = new IndexThread[numThreads[i]];
+ for (int x = 0; x < threads.Length; x++)
+ {
+ threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false);
+ threads[x].Start();
+ }
+
+ for (int x = 0; x < threads.Length; x++)
+ {
+ threads[x].Join();
+ }
+
+ Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due numThreads=" + numThreads[i]);
+ Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
+ Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
+ Assert.IsTrue(flushPolicy.PeakDocCountWithoutFlush <= iwc.MaxBufferedDocs, "peak bytes without flush exceeded watermark");
+ AssertActiveBytesAfter(flushControl);
+ writer.Dispose();
+ Assert.AreEqual(0, flushControl.ActiveBytes);
+ dir.Dispose();
+ }
+ }
+
+ [Test]
+ public virtual void TestRandom()
+ {
+ int numThreads = 1 + Random().Next(8);
+ int numDocumentsToIndex = 50 + AtLeast(70);
+ AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
+ Directory dir = NewDirectory();
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ MockDefaultFlushPolicy flushPolicy = new MockDefaultFlushPolicy();
+ iwc.SetFlushPolicy(flushPolicy);
+
+ int numDWPT = 1 + Random().Next(8);
+ DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numDWPT);
+ iwc.SetIndexerThreadPool(threadPool);
+
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ flushPolicy = (MockDefaultFlushPolicy)writer.Config.FlushPolicy;
+ DocumentsWriter docsWriter = writer.DocsWriter;
+ Assert.IsNotNull(docsWriter);
+ DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
+
+ Assert.AreEqual(0, flushControl.FlushBytes, " bytes must be 0 after init");
+
+ IndexThread[] threads = new IndexThread[numThreads];
+ for (int x = 0; x < threads.Length; x++)
+ {
+ threads[x] = new IndexThread(this, numDocs, numThreads, writer, LineDocFile, true);
+ threads[x].Start();
+ }
+
+ for (int x = 0; x < threads.Length; x++)
+ {
+ threads[x].Join();
+ }
+ Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
+ Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
+ Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
+ if (flushPolicy.FlushOnRAM && !flushPolicy.FlushOnDocCount && !flushPolicy.FlushOnDeleteTerms)
+ {
+ long maxRAMBytes = (long)(iwc.RAMBufferSizeMB * 1024.0 * 1024.0);
+ Assert.IsTrue(flushPolicy.PeakBytesWithoutFlush <= maxRAMBytes, "peak bytes without flush exceeded watermark");
+ if (flushPolicy.HasMarkedPending)
+ {
+ assertTrue("max: " + maxRAMBytes + " " + flushControl.peakActiveBytes, maxRAMBytes <= flushControl.peakActiveBytes);
+ }
+ }
+ AssertActiveBytesAfter(flushControl);
+ writer.Commit();
+ Assert.AreEqual(0, flushControl.ActiveBytes);
+ IndexReader r = DirectoryReader.Open(dir);
+ Assert.AreEqual(numDocumentsToIndex, r.NumDocs);
+ Assert.AreEqual(numDocumentsToIndex, r.MaxDoc);
+ if (!flushPolicy.FlushOnRAM)
+ {
+ assertFalse("never stall if we don't flush on RAM", docsWriter.flushControl.stallControl.WasStalled);
+ assertFalse("never block if we don't flush on RAM", docsWriter.flushControl.stallControl.HasBlocked);
+ }
+ r.Dispose();
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestStallControl()
+ {
+
+ int[] numThreads = new int[] { 4 + Random().Next(8), 1 };
+ int numDocumentsToIndex = 50 + Random().Next(50);
+ for (int i = 0; i < numThreads.Length; i++)
+ {
+ AtomicInt32 numDocs = new AtomicInt32(numDocumentsToIndex);
+ MockDirectoryWrapper dir = NewMockDirectory();
+ // mock a very slow harddisk sometimes here so that flushing is very slow
+ dir.Throttling = MockDirectoryWrapper.Throttling_e.SOMETIMES;
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ iwc.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ iwc.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ FlushPolicy flushPolicy = new FlushByRamOrCountsPolicy();
+ iwc.SetFlushPolicy(flushPolicy);
+
+ DocumentsWriterPerThreadPool threadPool = new ThreadAffinityDocumentsWriterThreadPool(numThreads[i] == 1 ? 1 : 2);
+ iwc.SetIndexerThreadPool(threadPool);
+ // with such a small ram buffer we should be stalled quiet quickly
+ iwc.SetRAMBufferSizeMB(0.25);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ IndexThread[] threads = new IndexThread[numThreads[i]];
+ for (int x = 0; x < threads.Length; x++)
+ {
+ threads[x] = new IndexThread(this, numDocs, numThreads[i], writer, LineDocFile, false);
+ threads[x].Start();
+ }
+
+ for (int x = 0; x < threads.Length; x++)
+ {
+ threads[x].Join();
+ }
+ DocumentsWriter docsWriter = writer.DocsWriter;
+ Assert.IsNotNull(docsWriter);
+ DocumentsWriterFlushControl flushControl = docsWriter.flushControl;
+ Assert.AreEqual(0, flushControl.FlushBytes, " all flushes must be due");
+ Assert.AreEqual(numDocumentsToIndex, writer.NumDocs);
+ Assert.AreEqual(numDocumentsToIndex, writer.MaxDoc);
+ if (numThreads[i] == 1)
+ {
+ assertFalse("single thread must not block numThreads: " + numThreads[i], docsWriter.flushControl.stallControl.HasBlocked);
+ }
+ if (docsWriter.flushControl.peakNetBytes > (2d * iwc.RAMBufferSizeMB * 1024d * 1024d))
+ {
+ Assert.IsTrue(docsWriter.flushControl.stallControl.WasStalled);
+ }
+ AssertActiveBytesAfter(flushControl);
+ writer.Dispose(true);
+ dir.Dispose();
+ }
+ }
+
+ internal virtual void AssertActiveBytesAfter(DocumentsWriterFlushControl flushControl)
+ {
+ IEnumerator<ThreadState> allActiveThreads = flushControl.AllActiveThreadStates();
+ long bytesUsed = 0;
+ while (allActiveThreads.MoveNext())
+ {
+ ThreadState next = allActiveThreads.Current;
+ if (next.DocumentsWriterPerThread != null)
+ {
+ bytesUsed += next.DocumentsWriterPerThread.BytesUsed;
+ }
+ }
+ Assert.AreEqual(bytesUsed, flushControl.ActiveBytes);
+ }
+
+ public class IndexThread : ThreadClass
+ {
+ private readonly TestFlushByRamOrCountsPolicy OuterInstance;
+
+ internal IndexWriter Writer;
+ internal LiveIndexWriterConfig Iwc;
+ internal LineFileDocs Docs;
+ internal AtomicInt32 PendingDocs;
+ internal readonly bool DoRandomCommit;
+
+ public IndexThread(TestFlushByRamOrCountsPolicy outerInstance, AtomicInt32 pendingDocs, int numThreads, IndexWriter writer, LineFileDocs docs, bool doRandomCommit)
+ {
+ this.OuterInstance = outerInstance;
+ this.PendingDocs = pendingDocs;
+ this.Writer = writer;
+ Iwc = writer.Config;
+ this.Docs = docs;
+ this.DoRandomCommit = doRandomCommit;
+ }
+
+ public override void Run()
+ {
+ try
+ {
+ long ramSize = 0;
+ while (PendingDocs.DecrementAndGet() > -1)
+ {
+ Document doc = Docs.NextDoc();
+ Writer.AddDocument(doc);
+ long newRamSize = Writer.RamSizeInBytes();
+ if (newRamSize != ramSize)
+ {
+ ramSize = newRamSize;
+ }
+ if (DoRandomCommit)
+ {
+ if (Rarely())
+ {
+ Writer.Commit();
+ }
+ }
+ }
+ Writer.Commit();
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine("FAILED exc:");
+ Console.WriteLine(ex.StackTrace);
+ throw new Exception(ex.Message, ex);
+ }
+ }
+ }
+
+ private class MockDefaultFlushPolicy : FlushByRamOrCountsPolicy
+ {
+ internal long PeakBytesWithoutFlush = int.MinValue;
+ internal long PeakDocCountWithoutFlush = int.MinValue;
+ internal bool HasMarkedPending = false;
+
+ public override void OnDelete(DocumentsWriterFlushControl control, ThreadState state)
+ {
+ List<ThreadState> pending = new List<ThreadState>();
+ List<ThreadState> notPending = new List<ThreadState>();
+ FindPending(control, pending, notPending);
+ bool flushCurrent = state.IsFlushPending;
+ ThreadState toFlush;
+ if (state.IsFlushPending)
+ {
+ toFlush = state;
+ }
+ else if (FlushOnDeleteTerms && state.DocumentsWriterPerThread.NumDeleteTerms >= m_indexWriterConfig.MaxBufferedDeleteTerms)
+ {
+ toFlush = state;
+ }
+ else
+ {
+ toFlush = null;
+ }
+ base.OnDelete(control, state);
+ if (toFlush != null)
+ {
+ if (flushCurrent)
+ {
+ Assert.IsTrue(pending.Remove(toFlush));
+ }
+ else
+ {
+ Assert.IsTrue(notPending.Remove(toFlush));
+ }
+ Assert.IsTrue(toFlush.IsFlushPending);
+ HasMarkedPending = true;
+ }
+
+ foreach (ThreadState threadState in notPending)
+ {
+ Assert.IsFalse(threadState.IsFlushPending);
+ }
+ }
+
+ public override void OnInsert(DocumentsWriterFlushControl control, ThreadState state)
+ {
+ List<ThreadState> pending = new List<ThreadState>();
+ List<ThreadState> notPending = new List<ThreadState>();
+ FindPending(control, pending, notPending);
+ bool flushCurrent = state.IsFlushPending;
+ long activeBytes = control.ActiveBytes;
+ ThreadState toFlush;
+ if (state.IsFlushPending)
+ {
+ toFlush = state;
+ }
+ else if (FlushOnDocCount && state.DocumentsWriterPerThread.NumDocsInRAM >= m_indexWriterConfig.MaxBufferedDocs)
+ {
+ toFlush = state;
+ }
+ else if (FlushOnRAM && activeBytes >= (long)(m_indexWriterConfig.RAMBufferSizeMB * 1024.0 * 1024.0))
+ {
+ toFlush = FindLargestNonPendingWriter(control, state);
+ Assert.IsFalse(toFlush.IsFlushPending);
+ }
+ else
+ {
+ toFlush = null;
+ }
+ base.OnInsert(control, state);
+ if (toFlush != null)
+ {
+ if (flushCurrent)
+ {
+ Assert.IsTrue(pending.Remove(toFlush));
+ }
+ else
+ {
+ Assert.IsTrue(notPending.Remove(toFlush));
+ }
+ Assert.IsTrue(toFlush.IsFlushPending);
+ HasMarkedPending = true;
+ }
+ else
+ {
+ PeakBytesWithoutFlush = Math.Max(activeBytes, PeakBytesWithoutFlush);
+ PeakDocCountWithoutFlush = Math.Max(state.DocumentsWriterPerThread.NumDocsInRAM, PeakDocCountWithoutFlush);
+ }
+
+ foreach (ThreadState threadState in notPending)
+ {
+ Assert.IsFalse(threadState.IsFlushPending);
+ }
+ }
+ }
+
+ internal static void FindPending(DocumentsWriterFlushControl flushControl, List<ThreadState> pending, List<ThreadState> notPending)
+ {
+ IEnumerator<ThreadState> allActiveThreads = flushControl.AllActiveThreadStates();
+ while (allActiveThreads.MoveNext())
+ {
+ ThreadState next = allActiveThreads.Current;
+ if (next.IsFlushPending)
+ {
+ pending.Add(next);
+ }
+ else
+ {
+ notPending.Add(next);
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestForTooMuchCloning.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestForTooMuchCloning.cs b/src/Lucene.Net.Tests/Index/TestForTooMuchCloning.cs
new file mode 100644
index 0000000..4218d04
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestForTooMuchCloning.cs
@@ -0,0 +1,86 @@
+using System.Text;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using NUnit.Framework;
+ using BytesRef = Lucene.Net.Util.BytesRef;
+ using Document = Documents.Document;
+ using Field = Field;
+ using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+ using TermRangeQuery = Lucene.Net.Search.TermRangeQuery;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+ using TextField = TextField;
+ using TopDocs = Lucene.Net.Search.TopDocs;
+
+ [TestFixture]
+ public class TestForTooMuchCloning : LuceneTestCase
+ {
+ // Make sure we don't clone IndexInputs too frequently
+ // during merging:
+ [Test]
+ public virtual void Test()
+ {
+ // NOTE: if we see a fail on this test with "NestedPulsing" its because its
+ // reuse isnt perfect (but reasonable). see TestPulsingReuse.testNestedPulsing
+ // for more details
+ MockDirectoryWrapper dir = NewMockDirectory();
+ TieredMergePolicy tmp = new TieredMergePolicy();
+ tmp.MaxMergeAtOnce = 2;
+ RandomIndexWriter w = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(tmp));
+ const int numDocs = 20;
+ for (int docs = 0; docs < numDocs; docs++)
+ {
+ StringBuilder sb = new StringBuilder();
+ for (int terms = 0; terms < 100; terms++)
+ {
+ sb.Append(TestUtil.RandomRealisticUnicodeString(Random()));
+ sb.Append(' ');
+ }
+ Document doc = new Document();
+ doc.Add(new TextField("field", sb.ToString(), Field.Store.NO));
+ w.AddDocument(doc);
+ }
+ IndexReader r = w.Reader;
+ w.Dispose();
+
+ int cloneCount = dir.InputCloneCount;
+ //System.out.println("merge clone count=" + cloneCount);
+ Assert.IsTrue(cloneCount < 500, "too many calls to IndexInput.clone during merging: " + dir.InputCloneCount);
+
+ IndexSearcher s = NewSearcher(r);
+
+ // MTQ that matches all terms so the AUTO_REWRITE should
+ // cutover to filter rewrite and reuse a single DocsEnum
+ // across all terms;
+ TopDocs hits = s.Search(new TermRangeQuery("field", new BytesRef(), new BytesRef("\uFFFF"), true, true), 10);
+ Assert.IsTrue(hits.TotalHits > 0);
+ int queryCloneCount = dir.InputCloneCount - cloneCount;
+ //System.out.println("query clone count=" + queryCloneCount);
+ Assert.IsTrue(queryCloneCount < 50, "too many calls to IndexInput.clone during TermRangeQuery: " + queryCloneCount);
+ r.Dispose();
+ dir.Dispose();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestForceMergeForever.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestForceMergeForever.cs b/src/Lucene.Net.Tests/Index/TestForceMergeForever.cs
new file mode 100644
index 0000000..12145b1
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestForceMergeForever.cs
@@ -0,0 +1,144 @@
+using System;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Support;
+ using NUnit.Framework;
+ using Directory = Lucene.Net.Store.Directory;
+ using LineFileDocs = Lucene.Net.Util.LineFileDocs;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+
+ [TestFixture]
+ public class TestForceMergeForever : LuceneTestCase
+ {
+ // Just counts how many merges are done
+ private class MyIndexWriter : IndexWriter
+ {
+ internal AtomicInt32 MergeCount = new AtomicInt32();
+ internal bool First;
+
+ public MyIndexWriter(Directory dir, IndexWriterConfig conf)
+ : base(dir, conf)
+ {
+ }
+
+ public override void Merge(MergePolicy.OneMerge merge)
+ {
+ if (merge.MaxNumSegments != -1 && (First || merge.Segments.Count == 1))
+ {
+ First = false;
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: maxNumSegments merge");
+ }
+ MergeCount.IncrementAndGet();
+ }
+ base.Merge(merge);
+ }
+ }
+
+ [Test]
+ public virtual void Test()
+ {
+ Directory d = NewDirectory();
+ MockAnalyzer analyzer = new MockAnalyzer(Random());
+ analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
+
+ MyIndexWriter w = new MyIndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
+
+ // Try to make an index that requires merging:
+ w.Config.SetMaxBufferedDocs(TestUtil.NextInt(Random(), 2, 11));
+ int numStartDocs = AtLeast(20);
+ LineFileDocs docs = new LineFileDocs(Random(), DefaultCodecSupportsDocValues());
+ for (int docIDX = 0; docIDX < numStartDocs; docIDX++)
+ {
+ w.AddDocument(docs.NextDoc());
+ }
+ MergePolicy mp = w.Config.MergePolicy;
+ int mergeAtOnce = 1 + w.segmentInfos.Count;
+ if (mp is TieredMergePolicy)
+ {
+ ((TieredMergePolicy)mp).MaxMergeAtOnce = mergeAtOnce;
+ }
+ else if (mp is LogMergePolicy)
+ {
+ ((LogMergePolicy)mp).MergeFactor = mergeAtOnce;
+ }
+ else
+ {
+ // skip test
+ w.Dispose();
+ d.Dispose();
+ return;
+ }
+
+ AtomicBoolean doStop = new AtomicBoolean();
+ w.Config.SetMaxBufferedDocs(2);
+ ThreadClass t = new ThreadAnonymousInnerClassHelper(this, w, numStartDocs, docs, doStop);
+ t.Start();
+ w.ForceMerge(1);
+ doStop.Set(true);
+ t.Join();
+ Assert.IsTrue(w.MergeCount.Get() <= 1, "merge count is " + w.MergeCount.Get());
+ w.Dispose();
+ d.Dispose();
+ docs.Dispose();
+ }
+
+ private class ThreadAnonymousInnerClassHelper : ThreadClass
+ {
+ private readonly TestForceMergeForever OuterInstance;
+
+ private Lucene.Net.Index.TestForceMergeForever.MyIndexWriter w;
+ private int NumStartDocs;
+ private LineFileDocs Docs;
+ private AtomicBoolean DoStop;
+
+ public ThreadAnonymousInnerClassHelper(TestForceMergeForever outerInstance, Lucene.Net.Index.TestForceMergeForever.MyIndexWriter w, int numStartDocs, LineFileDocs docs, AtomicBoolean doStop)
+ {
+ this.OuterInstance = outerInstance;
+ this.w = w;
+ this.NumStartDocs = numStartDocs;
+ this.Docs = docs;
+ this.DoStop = doStop;
+ }
+
+ public override void Run()
+ {
+ try
+ {
+ while (!DoStop.Get())
+ {
+ w.UpdateDocument(new Term("docid", "" + Random().Next(NumStartDocs)), Docs.NextDoc());
+ // Force deletes to apply
+ w.Reader.Dispose();
+ }
+ }
+ catch (Exception t)
+ {
+ throw new Exception(t.Message, t);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestIndexCommit.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestIndexCommit.cs b/src/Lucene.Net.Tests/Index/TestIndexCommit.cs
new file mode 100644
index 0000000..2cf47cb
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestIndexCommit.cs
@@ -0,0 +1,191 @@
+using System.Collections.Generic;
+
+namespace Lucene.Net.Index
+{
+ using NUnit.Framework;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using Directory = Lucene.Net.Store.Directory;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ [TestFixture]
+ public class TestIndexCommit : LuceneTestCase
+ {
+ [Test]
+ public virtual void TestEqualsHashCode()
+ {
+ // LUCENE-2417: equals and hashCode() impl was inconsistent
+ Directory dir = NewDirectory();
+
+ IndexCommit ic1 = new IndexCommitAnonymousInnerClassHelper(this, dir);
+
+ IndexCommit ic2 = new IndexCommitAnonymousInnerClassHelper2(this, dir);
+
+ Assert.AreEqual(ic1, ic2);
+ Assert.AreEqual(ic1.GetHashCode(), ic2.GetHashCode(), "hash codes are not equals");
+ dir.Dispose();
+ }
+
+ private class IndexCommitAnonymousInnerClassHelper : IndexCommit
+ {
+ private readonly TestIndexCommit OuterInstance;
+
+ private Directory Dir;
+
+ public IndexCommitAnonymousInnerClassHelper(TestIndexCommit outerInstance, Directory dir)
+ {
+ this.OuterInstance = outerInstance;
+ this.Dir = dir;
+ }
+
+ public override string SegmentsFileName
+ {
+ get
+ {
+ return "a";
+ }
+ }
+
+ public override Directory Directory
+ {
+ get
+ {
+ return Dir;
+ }
+ }
+
+ public override ICollection<string> FileNames
+ {
+ get
+ {
+ return null;
+ }
+ }
+
+ public override void Delete()
+ {
+ }
+
+ public override long Generation
+ {
+ get
+ {
+ return 0;
+ }
+ }
+
+ public override IDictionary<string, string> UserData
+ {
+ get
+ {
+ return null;
+ }
+ }
+
+ public override bool IsDeleted
+ {
+ get
+ {
+ return false;
+ }
+ }
+
+ public override int SegmentCount
+ {
+ get
+ {
+ return 2;
+ }
+ }
+ }
+
+ private class IndexCommitAnonymousInnerClassHelper2 : IndexCommit
+ {
+ private readonly TestIndexCommit OuterInstance;
+
+ private Directory Dir;
+
+ public IndexCommitAnonymousInnerClassHelper2(TestIndexCommit outerInstance, Directory dir)
+ {
+ this.OuterInstance = outerInstance;
+ this.Dir = dir;
+ }
+
+ public override string SegmentsFileName
+ {
+ get
+ {
+ return "b";
+ }
+ }
+
+ public override Directory Directory
+ {
+ get
+ {
+ return Dir;
+ }
+ }
+
+ public override ICollection<string> FileNames
+ {
+ get
+ {
+ return null;
+ }
+ }
+
+ public override void Delete()
+ {
+ }
+
+ public override long Generation
+ {
+ get
+ {
+ return 0;
+ }
+ }
+
+ public override IDictionary<string, string> UserData
+ {
+ get
+ {
+ return null;
+ }
+ }
+
+ public override bool IsDeleted
+ {
+ get
+ {
+ return false;
+ }
+ }
+
+ public override int SegmentCount
+ {
+ get
+ {
+ return 2;
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestIndexFileDeleter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestIndexFileDeleter.cs b/src/Lucene.Net.Tests/Index/TestIndexFileDeleter.cs
new file mode 100644
index 0000000..7b83e68
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestIndexFileDeleter.cs
@@ -0,0 +1,218 @@
+using Lucene.Net.Documents;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Index
+{
+ using NUnit.Framework;
+ using Codec = Lucene.Net.Codecs.Codec;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using Field = Field;
+ using IndexInput = Lucene.Net.Store.IndexInput;
+ using IndexOutput = Lucene.Net.Store.IndexOutput;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+
+ /*
+ Verify we can read the pre-2.1 file format, do searches
+ against it, and add documents to it.
+ */
+
+ [TestFixture]
+ public class TestIndexFileDeleter : LuceneTestCase
+ {
+ [Test]
+ public virtual void TestDeleteLeftoverFiles()
+ {
+ Directory dir = NewDirectory();
+ if (dir is MockDirectoryWrapper)
+ {
+ ((MockDirectoryWrapper)dir).PreventDoubleWrite = false;
+ }
+
+ MergePolicy mergePolicy = NewLogMergePolicy(true, 10);
+
+ // this test expects all of its segments to be in CFS
+ mergePolicy.NoCFSRatio = 1.0;
+ mergePolicy.MaxCFSSegmentSizeMB = double.PositiveInfinity;
+
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(10).SetMergePolicy(mergePolicy).SetUseCompoundFile(true));
+
+ int i;
+ for (i = 0; i < 35; i++)
+ {
+ AddDoc(writer, i);
+ }
+ writer.Config.MergePolicy.NoCFSRatio = 0.0;
+ writer.Config.SetUseCompoundFile(false);
+ for (; i < 45; i++)
+ {
+ AddDoc(writer, i);
+ }
+ writer.Dispose();
+
+ // Delete one doc so we get a .del file:
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).SetUseCompoundFile(true));
+ Term searchTerm = new Term("id", "7");
+ writer.DeleteDocuments(searchTerm);
+ writer.Dispose();
+
+ // Now, artificially create an extra .del file & extra
+ // .s0 file:
+ string[] files = dir.ListAll();
+
+ /*
+ for(int j=0;j<files.Length;j++) {
+ System.out.println(j + ": " + files[j]);
+ }
+ */
+
+ // TODO: fix this test better
+ string ext = Codec.Default.Name.Equals("SimpleText") ? ".liv" : ".del";
+
+ // Create a bogus separate del file for a
+ // segment that already has a separate del file:
+ CopyFile(dir, "_0_1" + ext, "_0_2" + ext);
+
+ // Create a bogus separate del file for a
+ // segment that does not yet have a separate del file:
+ CopyFile(dir, "_0_1" + ext, "_1_1" + ext);
+
+ // Create a bogus separate del file for a
+ // non-existent segment:
+ CopyFile(dir, "_0_1" + ext, "_188_1" + ext);
+
+ // Create a bogus segment file:
+ CopyFile(dir, "_0.cfs", "_188.cfs");
+
+ // Create a bogus fnm file when the CFS already exists:
+ CopyFile(dir, "_0.cfs", "_0.fnm");
+
+ // Create some old segments file:
+ CopyFile(dir, "segments_2", "segments");
+ CopyFile(dir, "segments_2", "segments_1");
+
+ // Create a bogus cfs file shadowing a non-cfs segment:
+
+ // TODO: assert is bogus (relies upon codec-specific filenames)
+ Assert.IsTrue(SlowFileExists(dir, "_3.fdt") || SlowFileExists(dir, "_3.fld"));
+ Assert.IsTrue(!SlowFileExists(dir, "_3.cfs"));
+ CopyFile(dir, "_1.cfs", "_3.cfs");
+
+ string[] filesPre = dir.ListAll();
+
+ // Open & close a writer: it should delete the above 4
+ // files and nothing more:
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND));
+ writer.Dispose();
+
+ string[] files2 = dir.ListAll();
+ dir.Dispose();
+
+ Array.Sort(files);
+ Array.Sort(files2);
+
+ HashSet<string> dif = DifFiles(files, files2);
+
+ if (!Arrays.Equals(files, files2))
+ {
+ Assert.Fail("IndexFileDeleter failed to delete unreferenced extra files: should have deleted " + (filesPre.Length - files.Length) + " files but only deleted " + (filesPre.Length - files2.Length) + "; expected files:\n " + AsString(files) + "\n actual files:\n " + AsString(files2) + "\ndiff: " + dif);
+ }
+ }
+
+ private static HashSet<string> DifFiles(string[] files1, string[] files2)
+ {
+ HashSet<string> set1 = new HashSet<string>();
+ HashSet<string> set2 = new HashSet<string>();
+ HashSet<string> extra = new HashSet<string>();
+
+ for (int x = 0; x < files1.Length; x++)
+ {
+ set1.Add(files1[x]);
+ }
+ for (int x = 0; x < files2.Length; x++)
+ {
+ set2.Add(files2[x]);
+ }
+ IEnumerator<string> i1 = set1.GetEnumerator();
+ while (i1.MoveNext())
+ {
+ string o = i1.Current;
+ if (!set2.Contains(o))
+ {
+ extra.Add(o);
+ }
+ }
+ IEnumerator<string> i2 = set2.GetEnumerator();
+ while (i2.MoveNext())
+ {
+ string o = i2.Current;
+ if (!set1.Contains(o))
+ {
+ extra.Add(o);
+ }
+ }
+ return extra;
+ }
+
+ private string AsString(string[] l)
+ {
+ string s = "";
+ for (int i = 0; i < l.Length; i++)
+ {
+ if (i > 0)
+ {
+ s += "\n ";
+ }
+ s += l[i];
+ }
+ return s;
+ }
+
+ public virtual void CopyFile(Directory dir, string src, string dest)
+ {
+ IndexInput @in = dir.OpenInput(src, NewIOContext(Random()));
+ IndexOutput @out = dir.CreateOutput(dest, NewIOContext(Random()));
+ var b = new byte[1024];
+ long remainder = @in.Length;
+ while (remainder > 0)
+ {
+ int len = (int)Math.Min(b.Length, remainder);
+ @in.ReadBytes(b, 0, len);
+ @out.WriteBytes(b, len);
+ remainder -= len;
+ }
+ @in.Dispose();
+ @out.Dispose();
+ }
+
+ private void AddDoc(IndexWriter writer, int id)
+ {
+ Document doc = new Document();
+ doc.Add(NewTextField("content", "aaa", Field.Store.NO));
+ doc.Add(NewStringField("id", Convert.ToString(id), Field.Store.NO));
+ writer.AddDocument(doc);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestIndexInput.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestIndexInput.cs b/src/Lucene.Net.Tests/Index/TestIndexInput.cs
new file mode 100644
index 0000000..32cd6a5
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestIndexInput.cs
@@ -0,0 +1,186 @@
+using System;
+
+namespace Lucene.Net.Index
+{
+ using NUnit.Framework;
+ using System.IO;
+ using System.Reflection;
+ using ByteArrayDataInput = Lucene.Net.Store.ByteArrayDataInput;
+ using ByteArrayDataOutput = Lucene.Net.Store.ByteArrayDataOutput;
+ using DataInput = Lucene.Net.Store.DataInput;
+ using IndexInput = Lucene.Net.Store.IndexInput;
+ using IndexOutput = Lucene.Net.Store.IndexOutput;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+ using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+
+ [TestFixture]
+ public class TestIndexInput : LuceneTestCase
+ {
+ internal static readonly byte[] READ_TEST_BYTES = new byte[] { unchecked((byte)(sbyte)0x80), 0x01, unchecked((byte)(sbyte)0xFF), 0x7F, unchecked((byte)(sbyte)0x80), unchecked((byte)(sbyte)0x80), 0x01, unchecked((byte)(sbyte)0x81), unchecked((byte)(sbyte)0x80), 0x01, unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), 0x07, unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), 0x0F, unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), 0x07, unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), (byte)0x7F, 0x06, (byte)'L', (byte)'u', (byte)'c', (byte)'e', (byte)'n', (byte)'e', 0x02, unchecked((byt
e)(sbyte)0xC2), unchecked((byte)(sbyte)0xBF), 0x0A, (byte)'L', (byte)'u', unchecked((byte)(sbyte)0xC2), unchecked((byte)(sbyte)0xBF), (byte)(sbyte)'c', (byte)'e', unchecked((byte)(sbyte)0xC2), unchecked((byte)(sbyte)0xBF), (byte)'n', (byte)'e', 0x03, unchecked((byte)(sbyte)0xE2), unchecked((byte)(sbyte)0x98), unchecked((byte)(sbyte)0xA0), 0x0C, (byte)'L', (byte)'u', unchecked((byte)(sbyte)0xE2), unchecked((byte)(sbyte)0x98), unchecked((byte)(sbyte)0xA0), (byte)'c', (byte)'e', unchecked((byte)(sbyte)0xE2), unchecked((byte)(sbyte)0x98), unchecked((byte)(sbyte)0xA0), (byte)'n', (byte)'e', 0x04, unchecked((byte)(sbyte)0xF0), unchecked((byte)(sbyte)0x9D), unchecked((byte)(sbyte)0x84), unchecked((byte)(sbyte)0x9E), 0x08, unchecked((byte)(sbyte)0xF0), unchecked((byte)(sbyte)0x9D), unchecked((byte)(sbyte)0x84), unchecked((byte)(sbyte)0x9E), unchecked((byte)(sbyte)0xF0), unchecked((byte)(sbyte)0x9D), unchecked((byte)(sbyte)0x85), unchecked((byte)(sbyte)0xA0), 0x0E, (byte)'L', (byte)'u', unch
ecked((byte)(sbyte)0xF0), unchecked((byte)(sbyte)0x9D), unchecked((byte)(sbyte)0x84), unchecked((byte)(sbyte)0x9E), (byte)'c', (byte)'e', unchecked((byte)(sbyte)0xF0), unchecked((byte)(sbyte)0x9D), unchecked((byte)(sbyte)0x85), unchecked((byte)(sbyte)0xA0), (byte)'n', (byte)'e', 0x01, 0x00, 0x08, (byte)'L', (byte)'u', 0x00, (byte)'c', (byte)'e', 0x00, (byte)'n', (byte)'e', unchecked((byte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), (byte)0x17, (byte)0x01, unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), unchecked((byte)(sbyte)0xFF), 0x01 };
+
+ internal static readonly int COUNT = RANDOM_MULTIPLIER * 65536;
+ internal static int[] INTS;
+ internal static long[] LONGS;
+ internal static byte[] RANDOM_TEST_BYTES;
+
+ [OneTimeSetUp]
+ public static void BeforeClass()
+ {
+ Random random = Random();
+ INTS = new int[COUNT];
+ LONGS = new long[COUNT];
+ RANDOM_TEST_BYTES = new byte[COUNT * (5 + 4 + 9 + 8)];
+ ByteArrayDataOutput bdo = new ByteArrayDataOutput(RANDOM_TEST_BYTES);
+ for (int i = 0; i < COUNT; i++)
+ {
+ int i1 = INTS[i] = random.Next();
+ bdo.WriteVInt32(i1);
+ bdo.WriteInt32(i1);
+
+ long l1;
+ if (Rarely())
+ {
+ // a long with lots of zeroes at the end
+ l1 = LONGS[i] = TestUtil.NextLong(random, 0, int.MaxValue) << 32;
+ }
+ else
+ {
+ l1 = LONGS[i] = TestUtil.NextLong(random, 0, long.MaxValue);
+ }
+ bdo.WriteVInt64(l1);
+ bdo.WriteInt64(l1);
+ }
+ }
+
+ [OneTimeTearDown]
+ public static void AfterClass()
+ {
+ INTS = null;
+ LONGS = null;
+ RANDOM_TEST_BYTES = null;
+ }
+
+ private void CheckReads(DataInput @is, Type expectedEx)
+ {
+ Assert.AreEqual(128, @is.ReadVInt32());
+ Assert.AreEqual(16383, @is.ReadVInt32());
+ Assert.AreEqual(16384, @is.ReadVInt32());
+ Assert.AreEqual(16385, @is.ReadVInt32());
+ Assert.AreEqual(int.MaxValue, @is.ReadVInt32());
+ Assert.AreEqual(-1, @is.ReadVInt32());
+ Assert.AreEqual((long)int.MaxValue, @is.ReadVInt64());
+ Assert.AreEqual(long.MaxValue, @is.ReadVInt64());
+ Assert.AreEqual("Lucene", @is.ReadString());
+
+ Assert.AreEqual("\u00BF", @is.ReadString());
+ Assert.AreEqual("Lu\u00BFce\u00BFne", @is.ReadString());
+
+ Assert.AreEqual("\u2620", @is.ReadString());
+ Assert.AreEqual("Lu\u2620ce\u2620ne", @is.ReadString());
+
+ Assert.AreEqual("\uD834\uDD1E", @is.ReadString());
+ Assert.AreEqual("\uD834\uDD1E\uD834\uDD60", @is.ReadString());
+ Assert.AreEqual("Lu\uD834\uDD1Ece\uD834\uDD60ne", @is.ReadString());
+
+ Assert.AreEqual("\u0000", @is.ReadString());
+ Assert.AreEqual("Lu\u0000ce\u0000ne", @is.ReadString());
+
+ try
+ {
+ @is.ReadVInt32();
+ Assert.Fail("Should throw " + expectedEx.Name);
+ }
+ catch (Exception e)
+ {
+ Assert.IsTrue(e.Message.StartsWith("Invalid vInt"));
+ Assert.IsTrue(expectedEx.IsInstanceOfType(e));
+ }
+ Assert.AreEqual(1, @is.ReadVInt32()); // guard value
+
+ try
+ {
+ @is.ReadVInt64();
+ Assert.Fail("Should throw " + expectedEx.Name);
+ }
+ catch (Exception e)
+ {
+ Assert.IsTrue(e.Message.StartsWith("Invalid vLong"));
+ Assert.IsTrue(expectedEx.IsInstanceOfType(e));
+ }
+ Assert.AreEqual(1L, @is.ReadVInt64()); // guard value
+ }
+
+ private void CheckRandomReads(DataInput @is)
+ {
+ for (int i = 0; i < COUNT; i++)
+ {
+ Assert.AreEqual(INTS[i], @is.ReadVInt32());
+ Assert.AreEqual(INTS[i], @is.ReadInt32());
+ Assert.AreEqual(LONGS[i], @is.ReadVInt64());
+ Assert.AreEqual(LONGS[i], @is.ReadInt64());
+ }
+ }
+
+ // this test only checks BufferedIndexInput because MockIndexInput extends BufferedIndexInput
+ [Test]
+ public virtual void TestBufferedIndexInputRead()
+ {
+ IndexInput @is = new MockIndexInput(READ_TEST_BYTES);
+ CheckReads(@is, typeof(IOException));
+ @is.Dispose();
+ @is = new MockIndexInput(RANDOM_TEST_BYTES);
+ CheckRandomReads(@is);
+ @is.Dispose();
+ }
+
+ // this test checks the raw IndexInput methods as it uses RAMIndexInput which extends IndexInput directly
+ [Test]
+ public virtual void TestRawIndexInputRead()
+ {
+ Random random = Random();
+ RAMDirectory dir = new RAMDirectory();
+ IndexOutput os = dir.CreateOutput("foo", NewIOContext(random));
+ os.WriteBytes(READ_TEST_BYTES, READ_TEST_BYTES.Length);
+ os.Dispose();
+ IndexInput @is = dir.OpenInput("foo", NewIOContext(random));
+ CheckReads(@is, typeof(IOException));
+ @is.Dispose();
+
+ os = dir.CreateOutput("bar", NewIOContext(random));
+ os.WriteBytes(RANDOM_TEST_BYTES, RANDOM_TEST_BYTES.Length);
+ os.Dispose();
+ @is = dir.OpenInput("bar", NewIOContext(random));
+ CheckRandomReads(@is);
+ @is.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestByteArrayDataInput()
+ {
+ ByteArrayDataInput @is = new ByteArrayDataInput((byte[])(Array)READ_TEST_BYTES);
+ CheckReads(@is, typeof(Exception));
+ @is = new ByteArrayDataInput(RANDOM_TEST_BYTES);
+ CheckRandomReads(@is);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestIndexReaderClose.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestIndexReaderClose.cs b/src/Lucene.Net.Tests/Index/TestIndexReaderClose.cs
new file mode 100644
index 0000000..9c96ced
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestIndexReaderClose.cs
@@ -0,0 +1,155 @@
+using System.Collections.Generic;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using Lucene.Net.Support;
+ using NUnit.Framework;
+ using System;
+ using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
+ using Directory = Lucene.Net.Store.Directory;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+
+ [TestFixture]
+ public class TestIndexReaderClose : LuceneTestCase
+ {
+ [Test]
+ public virtual void TestCloseUnderException()
+ {
+ int iters = 1000 + 1 + Random().nextInt(20);
+ for (int j = 0; j < iters; j++)
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir,
+ NewIndexWriterConfig(Random(), TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ writer.Commit();
+ writer.Dispose();
+ DirectoryReader open = DirectoryReader.Open(dir);
+ bool throwOnClose = !Rarely();
+ AtomicReader wrap = SlowCompositeReaderWrapper.Wrap(open);
+ FilterAtomicReader reader = new FilterAtomicReaderAnonymousInnerClassHelper(this, wrap, throwOnClose);
+ IList<IndexReader.IReaderClosedListener> listeners = new List<IndexReader.IReaderClosedListener>();
+ int listenerCount = Random().Next(20);
+ AtomicInt32 count = new AtomicInt32();
+ bool faultySet = false;
+ for (int i = 0; i < listenerCount; i++)
+ {
+ if (Rarely())
+ {
+ faultySet = true;
+ reader.AddReaderClosedListener(new FaultyListener());
+ }
+ else
+ {
+ count.IncrementAndGet();
+ reader.AddReaderClosedListener(new CountListener(count));
+ }
+ }
+ if (!faultySet && !throwOnClose)
+ {
+ reader.AddReaderClosedListener(new FaultyListener());
+ }
+ try
+ {
+ reader.Dispose();
+ Assert.Fail("expected Exception");
+ }
+ catch (InvalidOperationException ex)
+ {
+ if (throwOnClose)
+ {
+ Assert.AreEqual("BOOM!", ex.Message);
+ }
+ else
+ {
+ Assert.AreEqual("GRRRRRRRRRRRR!", ex.Message);
+ }
+ }
+
+ try
+ {
+ var aaa = reader.Fields;
+ Assert.Fail("we are closed");
+ }
+#pragma warning disable 168
+ catch (AlreadyClosedException ex)
+#pragma warning restore 168
+ {
+ }
+
+ if (Random().NextBoolean())
+ {
+ reader.Dispose(); // call it again
+ }
+ Assert.AreEqual(0, count.Get());
+ wrap.Dispose();
+ dir.Dispose();
+ }
+ }
+
+ private class FilterAtomicReaderAnonymousInnerClassHelper : FilterAtomicReader
+ {
+ private readonly TestIndexReaderClose OuterInstance;
+
+ private bool ThrowOnClose;
+
+ public FilterAtomicReaderAnonymousInnerClassHelper(TestIndexReaderClose outerInstance, AtomicReader wrap, bool throwOnClose)
+ : base(wrap)
+ {
+ this.OuterInstance = outerInstance;
+ this.ThrowOnClose = throwOnClose;
+ }
+
+ protected internal override void DoClose()
+ {
+ base.DoClose();
+ if (ThrowOnClose)
+ {
+ throw new InvalidOperationException("BOOM!");
+ }
+ }
+ }
+
+ private sealed class CountListener : IndexReader.IReaderClosedListener
+ {
+ internal readonly AtomicInt32 Count;
+
+ public CountListener(AtomicInt32 count)
+ {
+ this.Count = count;
+ }
+
+ public void OnClose(IndexReader reader)
+ {
+ Count.DecrementAndGet();
+ }
+ }
+
+ private sealed class FaultyListener : IndexReader.IReaderClosedListener
+ {
+ public void OnClose(IndexReader reader)
+ {
+ throw new InvalidOperationException("GRRRRRRRRRRRR!");
+ }
+ }
+ }
+}
\ No newline at end of file