You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/02/26 23:37:15 UTC
[27/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests:
Removed \core directory and put its contents in root directory
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestIndexWriterOnJRECrash.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestIndexWriterOnJRECrash.cs b/src/Lucene.Net.Tests/Index/TestIndexWriterOnJRECrash.cs
new file mode 100644
index 0000000..4254df5
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestIndexWriterOnJRECrash.cs
@@ -0,0 +1,281 @@
+// LUCENENET NOTE: Clearly this test is not applicable to .NET, but just
+// adding the file to the project for completedness.
+
+//using System;
+//using System.Collections.Generic;
+//using System.Threading;
+//using Lucene.Net.Randomized;
+//using Lucene.Net.Randomized.Generators;
+
+//namespace Lucene.Net.Index
+//{
+
+// /*
+// /// Licensed to the Apache Software Foundation (ASF) under one or more
+// /// contributor license agreements. See the NOTICE file distributed with
+// /// this work for additional information regarding copyright ownership.
+// /// The ASF licenses this file to You under the Apache License, Version 2.0
+// /// (the "License"); you may not use this file except in compliance with
+// /// the License. You may obtain a copy of the License at
+// ///
+// /// http://www.apache.org/licenses/LICENSE-2.0
+// ///
+// /// Unless required by applicable law or agreed to in writing, software
+// /// distributed under the License is distributed on an "AS IS" BASIS,
+// /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// /// See the License for the specific language governing permissions and
+// /// limitations under the License.
+// ///
+// */
+
+
+// using Codec = Lucene.Net.Codecs.Codec;
+// using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper;
+// using Constants = Lucene.Net.Util.Constants;
+// using TestUtil = Lucene.Net.Util.TestUtil;
+
+// using NUnit.Framework;
+// using Lucene.Net.Support;
+// using System.IO;
+// /// <summary>
+// /// Runs TestNRTThreads in a separate process, crashes the JRE in the middle
+// /// of execution, then runs checkindex to make sure its not corrupt.
+// /// </summary>
+// [TestFixture]
+// public class TestIndexWriterOnJRECrash : TestNRTThreads
+// {
+// private DirectoryInfo TempDir;
+
+// [SetUp]
+// public override void SetUp()
+// {
+// base.SetUp();
+// TempDir = CreateTempDir("jrecrash");
+// TempDir.Delete();
+// TempDir.mkdir();
+// }
+
+// [Test]
+// public override void TestNRTThreads_Mem()
+// {
+// // if we are not the fork
+// if (System.getProperty("tests.crashmode") == null)
+// {
+// // try up to 10 times to create an index
+// for (int i = 0; i < 10; i++)
+// {
+// ForkTest();
+// // if we succeeded in finding an index, we are done.
+// if (CheckIndexes(TempDir))
+// {
+// return;
+// }
+// }
+// }
+// else
+// {
+// // TODO: the non-fork code could simply enable impersonation?
+// AssumeFalse("does not support PreFlex, see LUCENE-3992", Codec.Default.Name.Equals("Lucene3x"));
+// // we are the fork, setup a crashing thread
+// int crashTime = TestUtil.NextInt(Random(), 3000, 4000);
+// ThreadClass t = new ThreadAnonymousInnerClassHelper(this, crashTime);
+// t.Priority = ThreadPriority.Highest;
+// t.Start();
+// // run the test until we crash.
+// for (int i = 0; i < 1000; i++)
+// {
+// base.TestNRTThreads_Mem();
+// }
+// }
+// }
+
+// private class ThreadAnonymousInnerClassHelper : ThreadClass
+// {
+// private readonly TestIndexWriterOnJRECrash OuterInstance;
+
+// private int CrashTime;
+
+// public ThreadAnonymousInnerClassHelper(TestIndexWriterOnJRECrash outerInstance, int crashTime)
+// {
+// this.OuterInstance = outerInstance;
+// this.CrashTime = crashTime;
+// }
+
+// public override void Run()
+// {
+// try
+// {
+// Thread.Sleep(CrashTime);
+// }
+// catch (ThreadInterruptedException e)
+// {
+// }
+// OuterInstance.CrashJRE();
+// }
+// }
+
+// /// <summary>
+// /// fork ourselves in a new jvm. sets -Dtests.crashmode=true </summary>
+// public virtual void ForkTest()
+// {
+// IList<string> cmd = new List<string>();
+// cmd.Add(System.getProperty("java.home") + System.getProperty("file.separator") + "bin" + System.getProperty("file.separator") + "java");
+// cmd.Add("-Xmx512m");
+// cmd.Add("-Dtests.crashmode=true");
+// // passing NIGHTLY to this test makes it run for much longer, easier to catch it in the act...
+// cmd.Add("-Dtests.nightly=true");
+// cmd.Add("-DtempDir=" + TempDir.Path);
+// cmd.Add("-Dtests.seed=" + SeedUtils.formatSeed(Random().NextLong()));
+// cmd.Add("-ea");
+// cmd.Add("-cp");
+// cmd.Add(System.getProperty("java.class.path"));
+// cmd.Add("org.junit.runner.JUnitCore");
+// cmd.Add(this.GetType().Name);
+// ProcessBuilder pb = new ProcessBuilder(cmd);
+// pb.directory(TempDir);
+// pb.redirectErrorStream(true);
+// Process p = pb.Start();
+
+// // We pump everything to stderr.
+// PrintStream childOut = System.err;
+// Thread stdoutPumper = ThreadPumper.Start(p.InputStream, childOut);
+// Thread stderrPumper = ThreadPumper.Start(p.ErrorStream, childOut);
+// if (VERBOSE)
+// {
+// childOut.println(">>> Begin subprocess output");
+// }
+// p.waitFor();
+// stdoutPumper.Join();
+// stderrPumper.Join();
+// if (VERBOSE)
+// {
+// childOut.println("<<< End subprocess output");
+// }
+// }
+
+// /// <summary>
+// /// A pipe thread. It'd be nice to reuse guava's implementation for this... </summary>
+// internal class ThreadPumper
+// {
+// public static Thread Start(InputStream from, OutputStream to)
+// {
+// ThreadClass t = new ThreadAnonymousInnerClassHelper2(from, to);
+// t.Start();
+// return t;
+// }
+
+// private class ThreadAnonymousInnerClassHelper2 : ThreadClass
+// {
+// private InputStream From;
+// private OutputStream To;
+
+// public ThreadAnonymousInnerClassHelper2(InputStream from, OutputStream to)
+// {
+// this.From = from;
+// this.To = to;
+// }
+
+// public override void Run()
+// {
+// try
+// {
+// sbyte[] buffer = new sbyte[1024];
+// int len;
+// while ((len = From.Read(buffer)) != -1)
+// {
+// if (VERBOSE)
+// {
+// To.Write(buffer, 0, len);
+// }
+// }
+// }
+// catch (IOException e)
+// {
+// Console.Error.WriteLine("Couldn't pipe from the forked process: " + e.ToString());
+// }
+// }
+// }
+// }
+
+// /// <summary>
+// /// Recursively looks for indexes underneath <code>file</code>,
+// /// and runs checkindex on them. returns true if it found any indexes.
+// /// </summary>
+// public virtual bool CheckIndexes(DirectoryInfo file)
+// {
+// if (file.IsDirectory)
+// {
+// BaseDirectoryWrapper dir = NewFSDirectory(file);
+// dir.CheckIndexOnClose = false; // don't double-checkindex
+// if (DirectoryReader.IndexExists(dir))
+// {
+// if (VERBOSE)
+// {
+// Console.Error.WriteLine("Checking index: " + file);
+// }
+// // LUCENE-4738: if we crashed while writing first
+// // commit it's possible index will be corrupt (by
+// // design we don't try to be smart about this case
+// // since that too risky):
+// if (SegmentInfos.GetLastCommitGeneration(dir) > 1)
+// {
+// TestUtil.CheckIndex(dir);
+// }
+// dir.Dispose();
+// return true;
+// }
+// dir.Dispose();
+// foreach (FileInfo f in file.ListAll())
+// {
+// if (CheckIndexes(f))
+// {
+// return true;
+// }
+// }
+// }
+// return false;
+// }
+
+// /// <summary>
+// /// currently, this only works/tested on Sun and IBM.
+// /// </summary>
+// public virtual void CrashJRE()
+// {
+// string vendor = Constants.JAVA_VENDOR;
+// bool supportsUnsafeNpeDereference = vendor.StartsWith("Oracle") || vendor.StartsWith("Sun") || vendor.StartsWith("Apple");
+
+// try
+// {
+// if (supportsUnsafeNpeDereference)
+// {
+// try
+// {
+// Type clazz = Type.GetType("sun.misc.Unsafe");
+// Field field = clazz.GetDeclaredField("theUnsafe");
+// field.Accessible = true;
+// object o = field.Get(null);
+// Method m = clazz.GetMethod("putAddress", typeof(long), typeof(long));
+// m.invoke(o, 0L, 0L);
+// }
+// catch (Exception e)
+// {
+// Console.WriteLine("Couldn't kill the JVM via Unsafe.");
+// Console.WriteLine(e.StackTrace);
+// }
+// }
+
+// // Fallback attempt to Runtime.halt();
+// Runtime.Runtime.halt(-1);
+// }
+// catch (Exception e)
+// {
+// Console.WriteLine("Couldn't kill the JVM.");
+// Console.WriteLine(e.StackTrace);
+// }
+
+// // We couldn't get the JVM to crash for some reason.
+// Assert.Fail();
+// }
+// }
+
+//}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestIndexWriterOutOfFileDescriptors.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestIndexWriterOutOfFileDescriptors.cs b/src/Lucene.Net.Tests/Index/TestIndexWriterOutOfFileDescriptors.cs
new file mode 100644
index 0000000..24a3726
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestIndexWriterOutOfFileDescriptors.cs
@@ -0,0 +1,196 @@
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using NUnit.Framework;
+ using System.IO;
+ using Directory = Lucene.Net.Store.Directory;
+ using IOContext = Lucene.Net.Store.IOContext;
+ using LineFileDocs = Lucene.Net.Util.LineFileDocs;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+ using PrintStreamInfoStream = Lucene.Net.Util.PrintStreamInfoStream;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+
+ [TestFixture]
+ public class TestIndexWriterOutOfFileDescriptors : LuceneTestCase
+ {
+ [Test]
+ public virtual void Test()
+ {
+ MockDirectoryWrapper dir = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors"));
+ dir.PreventDoubleWrite = false;
+ double rate = Random().NextDouble() * 0.01;
+ //System.out.println("rate=" + rate);
+ dir.RandomIOExceptionRateOnOpen = rate;
+ int iters = AtLeast(20);
+ LineFileDocs docs = new LineFileDocs(Random(), DefaultCodecSupportsDocValues());
+ IndexReader r = null;
+ DirectoryReader r2 = null;
+ bool any = false;
+ MockDirectoryWrapper dirCopy = null;
+ int lastNumDocs = 0;
+ for (int iter = 0; iter < iters; iter++)
+ {
+ IndexWriter w = null;
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: iter=" + iter);
+ }
+ try
+ {
+ MockAnalyzer analyzer = new MockAnalyzer(Random());
+ analyzer.MaxTokenLength = TestUtil.NextInt(Random(), 1, IndexWriter.MAX_TERM_LENGTH);
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+
+ if (VERBOSE)
+ {
+ // Do this ourselves instead of relying on LTC so
+ // we see incrementing messageID:
+ iwc.InfoStream = new PrintStreamInfoStream(Console.Out);
+ }
+ var ms = iwc.MergeScheduler;
+ if (ms is IConcurrentMergeScheduler)
+ {
+ ((IConcurrentMergeScheduler)ms).SetSuppressExceptions();
+ }
+ w = new IndexWriter(dir, iwc);
+ if (r != null && Random().Next(5) == 3)
+ {
+ if (Random().NextBoolean())
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: addIndexes IR[]");
+ }
+ w.AddIndexes(new IndexReader[] { r });
+ }
+ else
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: addIndexes Directory[]");
+ }
+ w.AddIndexes(new Directory[] { dirCopy });
+ }
+ }
+ else
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: addDocument");
+ }
+ w.AddDocument(docs.NextDoc());
+ }
+ dir.RandomIOExceptionRateOnOpen = 0.0;
+ w.Dispose();
+ w = null;
+
+ // NOTE: this is O(N^2)! Only enable for temporary debugging:
+ //dir.setRandomIOExceptionRateOnOpen(0.0);
+ //TestUtil.CheckIndex(dir);
+ //dir.setRandomIOExceptionRateOnOpen(rate);
+
+ // Verify numDocs only increases, to catch IndexWriter
+ // accidentally deleting the index:
+ dir.RandomIOExceptionRateOnOpen = 0.0;
+ Assert.IsTrue(DirectoryReader.IndexExists(dir));
+ if (r2 == null)
+ {
+ r2 = DirectoryReader.Open(dir);
+ }
+ else
+ {
+ DirectoryReader r3 = DirectoryReader.OpenIfChanged(r2);
+ if (r3 != null)
+ {
+ r2.Dispose();
+ r2 = r3;
+ }
+ }
+ Assert.IsTrue(r2.NumDocs >= lastNumDocs, "before=" + lastNumDocs + " after=" + r2.NumDocs);
+ lastNumDocs = r2.NumDocs;
+ //System.out.println("numDocs=" + lastNumDocs);
+ dir.RandomIOExceptionRateOnOpen = rate;
+
+ any = true;
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: iter=" + iter + ": success");
+ }
+ }
+ catch (IOException ioe)
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: iter=" + iter + ": exception");
+ Console.WriteLine(ioe.ToString());
+ Console.Write(ioe.StackTrace);
+ }
+ if (w != null)
+ {
+ // NOTE: leave random IO exceptions enabled here,
+ // to verify that rollback does not try to write
+ // anything:
+ w.Rollback();
+ }
+ }
+
+ if (any && r == null && Random().NextBoolean())
+ {
+ // Make a copy of a non-empty index so we can use
+ // it to addIndexes later:
+ dir.RandomIOExceptionRateOnOpen = 0.0;
+ r = DirectoryReader.Open(dir);
+ dirCopy = NewMockFSDirectory(CreateTempDir("TestIndexWriterOutOfFileDescriptors.copy"));
+ HashSet<string> files = new HashSet<string>();
+ foreach (string file in dir.ListAll())
+ {
+ dir.Copy(dirCopy, file, file, IOContext.DEFAULT);
+ files.Add(file);
+ }
+ dirCopy.Sync(files);
+ // Have IW kiss the dir so we remove any leftover
+ // files ... we can easily have leftover files at
+ // the time we take a copy because we are holding
+ // open a reader:
+ (new IndexWriter(dirCopy, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())))).Dispose();
+ dirCopy.RandomIOExceptionRate = rate;
+ dir.RandomIOExceptionRateOnOpen = rate;
+ }
+ }
+
+ if (r2 != null)
+ {
+ r2.Dispose();
+ }
+ if (r != null)
+ {
+ r.Dispose();
+ dirCopy.Dispose();
+ }
+ dir.Dispose();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestIndexWriterReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestIndexWriterReader.cs b/src/Lucene.Net.Tests/Index/TestIndexWriterReader.cs
new file mode 100644
index 0000000..9207b98
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestIndexWriterReader.cs
@@ -0,0 +1,1419 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading;
+using Lucene.Net.Attributes;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using Lucene.Net.Support;
+ using NUnit.Framework;
+ using System.Collections.Concurrent;
+ using Util;
+ using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
+ using BytesRef = Lucene.Net.Util.BytesRef;
+ using Codec = Lucene.Net.Codecs.Codec;
+ using Directory = Lucene.Net.Store.Directory;
+ using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator;
+ using Document = Documents.Document;
+ using FakeIOException = Lucene.Net.Store.MockDirectoryWrapper.FakeIOException;
+ using Field = Field;
+ using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+ using InfoStream = Lucene.Net.Util.InfoStream;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ /// Copyright 2004 The Apache Software Foundation
+ ///
+ /// Licensed under the Apache License, Version 2.0 (the "License");
+ /// you may not use this file except in compliance with the License.
+ /// You may obtain a copy of the License at
+ ///
+ /// http://www.apache.org/licenses/LICENSE-2.0
+ ///
+ /// Unless required by applicable law or agreed to in writing, software
+ /// distributed under the License is distributed on an "AS IS" BASIS,
+ /// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ /// See the License for the specific language governing permissions and
+ /// limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+ using Query = Lucene.Net.Search.Query;
+ using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+ using TermQuery = Lucene.Net.Search.TermQuery;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+ using TextField = TextField;
+ using TopDocs = Lucene.Net.Search.TopDocs;
+
+ [TestFixture]
+ public class TestIndexWriterReader : LuceneTestCase
+ {
+ private readonly int NumThreads = TEST_NIGHTLY ? 5 : 3;
+
+ public static int Count(Term t, IndexReader r)
+ {
+ int count = 0;
+ DocsEnum td = TestUtil.Docs(Random(), r, t.Field, new BytesRef(t.Text()), MultiFields.GetLiveDocs(r), null, 0);
+
+ if (td != null)
+ {
+ while (td.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
+ {
+ var _ = td.DocID;
+ count++;
+ }
+ }
+ return count;
+ }
+
+ [Test]
+ public virtual void TestAddCloseOpen()
+ {
+ // Can't use assertNoDeletes: this test pulls a non-NRT
+ // reader in the end:
+ Directory dir1 = NewDirectory();
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+
+ IndexWriter writer = new IndexWriter(dir1, iwc);
+ for (int i = 0; i < 97; i++)
+ {
+ DirectoryReader reader = writer.Reader;
+ if (i == 0)
+ {
+ writer.AddDocument(DocHelper.CreateDocument(i, "x", 1 + Random().Next(5)));
+ }
+ else
+ {
+ int previous = Random().Next(i);
+ // a check if the reader is current here could fail since there might be
+ // merges going on.
+ switch (Random().Next(5))
+ {
+ case 0:
+ case 1:
+ case 2:
+ writer.AddDocument(DocHelper.CreateDocument(i, "x", 1 + Random().Next(5)));
+ break;
+
+ case 3:
+ writer.UpdateDocument(new Term("id", "" + previous), DocHelper.CreateDocument(previous, "x", 1 + Random().Next(5)));
+ break;
+
+ case 4:
+ writer.DeleteDocuments(new Term("id", "" + previous));
+ break;
+ }
+ }
+ Assert.IsFalse(reader.IsCurrent);
+ reader.Dispose();
+ }
+ writer.ForceMerge(1); // make sure all merging is done etc.
+ DirectoryReader dirReader = writer.Reader;
+ writer.Commit(); // no changes that are not visible to the reader
+ Assert.IsTrue(dirReader.IsCurrent);
+ writer.Dispose();
+ Assert.IsTrue(dirReader.IsCurrent); // all changes are visible to the reader
+ iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ writer = new IndexWriter(dir1, iwc);
+ Assert.IsTrue(dirReader.IsCurrent);
+ writer.AddDocument(DocHelper.CreateDocument(1, "x", 1 + Random().Next(5)));
+ Assert.IsTrue(dirReader.IsCurrent); // segments in ram but IW is different to the readers one
+ writer.Dispose();
+ Assert.IsFalse(dirReader.IsCurrent); // segments written
+ dirReader.Dispose();
+ dir1.Dispose();
+ }
+
+ [Test]
+ public virtual void TestUpdateDocument()
+ {
+ bool doFullMerge = true;
+
+ Directory dir1 = NewDirectory();
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ if (iwc.MaxBufferedDocs < 20)
+ {
+ iwc.SetMaxBufferedDocs(20);
+ }
+ // no merging
+ if (Random().NextBoolean())
+ {
+ iwc.SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
+ }
+ else
+ {
+ iwc.SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
+ }
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: make index");
+ }
+ IndexWriter writer = new IndexWriter(dir1, iwc);
+
+ // create the index
+ CreateIndexNoClose(!doFullMerge, "index1", writer);
+
+ // writer.Flush(false, true, true);
+
+ // get a reader
+ DirectoryReader r1 = writer.Reader;
+ Assert.IsTrue(r1.IsCurrent);
+
+ string id10 = r1.Document(10).GetField("id").GetStringValue();
+
+ Document newDoc = r1.Document(10);
+ newDoc.RemoveField("id");
+ newDoc.Add(NewStringField("id", Convert.ToString(8000), Field.Store.YES));
+ writer.UpdateDocument(new Term("id", id10), newDoc);
+ Assert.IsFalse(r1.IsCurrent);
+
+ DirectoryReader r2 = writer.Reader;
+ Assert.IsTrue(r2.IsCurrent);
+ Assert.AreEqual(0, Count(new Term("id", id10), r2));
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: verify id");
+ }
+ Assert.AreEqual(1, Count(new Term("id", Convert.ToString(8000)), r2));
+
+ r1.Dispose();
+ Assert.IsTrue(r2.IsCurrent);
+ writer.Dispose();
+ Assert.IsTrue(r2.IsCurrent);
+
+ DirectoryReader r3 = DirectoryReader.Open(dir1);
+ Assert.IsTrue(r3.IsCurrent);
+ Assert.IsTrue(r2.IsCurrent);
+ Assert.AreEqual(0, Count(new Term("id", id10), r3));
+ Assert.AreEqual(1, Count(new Term("id", Convert.ToString(8000)), r3));
+
+ writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "a b c", Field.Store.NO));
+ writer.AddDocument(doc);
+ Assert.IsTrue(r2.IsCurrent);
+ Assert.IsTrue(r3.IsCurrent);
+
+ writer.Dispose();
+
+ Assert.IsFalse(r2.IsCurrent);
+ Assert.IsTrue(!r3.IsCurrent);
+
+ r2.Dispose();
+ r3.Dispose();
+
+ dir1.Dispose();
+ }
+
+ [Test]
+ public virtual void TestIsCurrent()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "a b c", Field.Store.NO));
+ writer.AddDocument(doc);
+ writer.Dispose();
+
+ iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ writer = new IndexWriter(dir, iwc);
+ doc = new Document();
+ doc.Add(NewTextField("field", "a b c", Field.Store.NO));
+ DirectoryReader nrtReader = writer.Reader;
+ Assert.IsTrue(nrtReader.IsCurrent);
+ writer.AddDocument(doc);
+ Assert.IsFalse(nrtReader.IsCurrent); // should see the changes
+ writer.ForceMerge(1); // make sure we don't have a merge going on
+ Assert.IsFalse(nrtReader.IsCurrent);
+ nrtReader.Dispose();
+
+ DirectoryReader dirReader = DirectoryReader.Open(dir);
+ nrtReader = writer.Reader;
+
+ Assert.IsTrue(dirReader.IsCurrent);
+ Assert.IsTrue(nrtReader.IsCurrent); // nothing was committed yet so we are still current
+ Assert.AreEqual(2, nrtReader.MaxDoc); // sees the actual document added
+ Assert.AreEqual(1, dirReader.MaxDoc);
+ writer.Dispose(); // close is actually a commit both should see the changes
+ Assert.IsTrue(nrtReader.IsCurrent);
+ Assert.IsFalse(dirReader.IsCurrent); // this reader has been opened before the writer was closed / committed
+
+ dirReader.Dispose();
+ nrtReader.Dispose();
+ dir.Dispose();
+ }
+
+ /// <summary>
+ /// Test using IW.addIndexes
+ /// </summary>
+ [Test]
+ public virtual void TestAddIndexes()
+ {
+ bool doFullMerge = false;
+
+ Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ if (iwc.MaxBufferedDocs < 20)
+ {
+ iwc.SetMaxBufferedDocs(20);
+ }
+ // no merging
+ if (Random().NextBoolean())
+ {
+ iwc.SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
+ }
+ else
+ {
+ iwc.SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
+ }
+ IndexWriter writer = new IndexWriter(dir1, iwc);
+
+ // create the index
+ CreateIndexNoClose(!doFullMerge, "index1", writer);
+ writer.Flush(false, true);
+
+ // create a 2nd index
+ Directory dir2 = NewDirectory();
+ IndexWriter writer2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ CreateIndexNoClose(!doFullMerge, "index2", writer2);
+ writer2.Dispose();
+
+ DirectoryReader r0 = writer.Reader;
+ Assert.IsTrue(r0.IsCurrent);
+ writer.AddIndexes(dir2);
+ Assert.IsFalse(r0.IsCurrent);
+ r0.Dispose();
+
+ DirectoryReader r1 = writer.Reader;
+ Assert.IsTrue(r1.IsCurrent);
+
+ writer.Commit();
+ Assert.IsTrue(r1.IsCurrent); // we have seen all changes - no change after opening the NRT reader
+
+ Assert.AreEqual(200, r1.MaxDoc);
+
+ int index2df = r1.DocFreq(new Term("indexname", "index2"));
+
+ Assert.AreEqual(100, index2df);
+
+ // verify the docs are from different indexes
+ Document doc5 = r1.Document(5);
+ Assert.AreEqual("index1", doc5.Get("indexname"));
+ Document doc150 = r1.Document(150);
+ Assert.AreEqual("index2", doc150.Get("indexname"));
+ r1.Dispose();
+ writer.Dispose();
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void ExposeCompTermVR()
+ {
+ bool doFullMerge = false;
+ Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ if (iwc.MaxBufferedDocs < 20)
+ {
+ iwc.SetMaxBufferedDocs(20);
+ }
+ // no merging
+ if (Random().NextBoolean())
+ {
+ iwc.SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES);
+ }
+ else
+ {
+ iwc.SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
+ }
+ IndexWriter writer = new IndexWriter(dir1, iwc);
+ CreateIndexNoClose(!doFullMerge, "index1", writer);
+ writer.Dispose();
+ dir1.Dispose();
+ }
+
+ [Test]
+ public virtual void TestAddIndexes2()
+ {
+ bool doFullMerge = false;
+
+ Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
+ IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ // create a 2nd index
+ Directory dir2 = NewDirectory();
+ IndexWriter writer2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ CreateIndexNoClose(!doFullMerge, "index2", writer2);
+ writer2.Dispose();
+
+ writer.AddIndexes(dir2);
+ writer.AddIndexes(dir2);
+ writer.AddIndexes(dir2);
+ writer.AddIndexes(dir2);
+ writer.AddIndexes(dir2);
+
+ IndexReader r1 = writer.Reader;
+ Assert.AreEqual(500, r1.MaxDoc);
+
+ r1.Dispose();
+ writer.Dispose();
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ /// <summary>
+ /// Deletes using IW.deleteDocuments
+ /// </summary>
+ [Test]
+ public virtual void TestDeleteFromIndexWriter()
+ {
+ bool doFullMerge = true;
+
+ Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
+ IndexWriter writer = new IndexWriter(dir1, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetReaderTermsIndexDivisor(2));
+ // create the index
+ CreateIndexNoClose(!doFullMerge, "index1", writer);
+ writer.Flush(false, true);
+ // get a reader
+ IndexReader r1 = writer.Reader;
+
+ string id10 = r1.Document(10).GetField("id").GetStringValue();
+
+ // deleted IW docs should not show up in the next getReader
+ writer.DeleteDocuments(new Term("id", id10));
+ IndexReader r2 = writer.Reader;
+ Assert.AreEqual(1, Count(new Term("id", id10), r1));
+ Assert.AreEqual(0, Count(new Term("id", id10), r2));
+
+ string id50 = r1.Document(50).GetField("id").GetStringValue();
+ Assert.AreEqual(1, Count(new Term("id", id50), r1));
+
+ writer.DeleteDocuments(new Term("id", id50));
+
+ IndexReader r3 = writer.Reader;
+ Assert.AreEqual(0, Count(new Term("id", id10), r3));
+ Assert.AreEqual(0, Count(new Term("id", id50), r3));
+
+ string id75 = r1.Document(75).GetField("id").GetStringValue();
+ writer.DeleteDocuments(new TermQuery(new Term("id", id75)));
+ IndexReader r4 = writer.Reader;
+ Assert.AreEqual(1, Count(new Term("id", id75), r3));
+ Assert.AreEqual(0, Count(new Term("id", id75), r4));
+
+ r1.Dispose();
+ r2.Dispose();
+ r3.Dispose();
+ r4.Dispose();
+ writer.Dispose();
+
+ // reopen the writer to verify the delete made it to the directory
+ writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ IndexReader w2r1 = writer.Reader;
+ Assert.AreEqual(0, Count(new Term("id", id10), w2r1));
+ w2r1.Dispose();
+ writer.Dispose();
+ dir1.Dispose();
+ }
+
+ [Test]
+ public virtual void TestAddIndexesAndDoDeletesThreads()
+ {
+ const int numIter = 2;
+ int numDirs = 3;
+
+ Directory mainDir = GetAssertNoDeletesDirectory(NewDirectory());
+
+ IndexWriter mainWriter = new IndexWriter(mainDir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));
+ TestUtil.ReduceOpenFiles(mainWriter);
+
+ AddDirectoriesThreads addDirThreads = new AddDirectoriesThreads(this, numIter, mainWriter);
+ addDirThreads.LaunchThreads(numDirs);
+ addDirThreads.JoinThreads();
+
+ //Assert.AreEqual(100 + numDirs * (3 * numIter / 4) * addDirThreads.numThreads
+ // * addDirThreads.NUM_INIT_DOCS, addDirThreads.mainWriter.NumDocs);
+ Assert.AreEqual(addDirThreads.Count.Get(), addDirThreads.MainWriter.NumDocs);
+
+ addDirThreads.Close(true);
+
+ Assert.IsTrue(addDirThreads.Failures.Count == 0);
+
+ TestUtil.CheckIndex(mainDir);
+
+ IndexReader reader = DirectoryReader.Open(mainDir);
+ Assert.AreEqual(addDirThreads.Count.Get(), reader.NumDocs);
+ //Assert.AreEqual(100 + numDirs * (3 * numIter / 4) * addDirThreads.numThreads
+ // * addDirThreads.NUM_INIT_DOCS, reader.NumDocs);
+ reader.Dispose();
+
+ addDirThreads.CloseDir();
+ mainDir.Dispose();
+ }
+
+ private class AddDirectoriesThreads
+ {
+ internal virtual void InitializeInstanceFields()
+ {
+ Threads = new ThreadClass[OuterInstance.NumThreads];
+ }
+
+ private readonly TestIndexWriterReader OuterInstance;
+
+ internal Directory AddDir;
+ internal const int NUM_INIT_DOCS = 100;
+ internal int NumDirs;
+ internal ThreadClass[] Threads;
+ internal IndexWriter MainWriter;
+ internal readonly IList<Exception> Failures = new List<Exception>();
+ internal IndexReader[] Readers;
+ internal bool DidClose = false;
+ internal AtomicInt32 Count = new AtomicInt32(0);
+ internal AtomicInt32 NumaddIndexes = new AtomicInt32(0);
+
+ public AddDirectoriesThreads(TestIndexWriterReader outerInstance, int numDirs, IndexWriter mainWriter)
+ {
+ this.OuterInstance = outerInstance;
+
+ InitializeInstanceFields();
+ this.NumDirs = numDirs;
+ this.MainWriter = mainWriter;
+ AddDir = NewDirectory();
+ IndexWriter writer = new IndexWriter(AddDir, OuterInstance.NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
+ TestUtil.ReduceOpenFiles(writer);
+ for (int i = 0; i < NUM_INIT_DOCS; i++)
+ {
+ Document doc = DocHelper.CreateDocument(i, "addindex", 4);
+ writer.AddDocument(doc);
+ }
+
+ writer.Dispose();
+
+ Readers = new IndexReader[numDirs];
+ for (int i = 0; i < numDirs; i++)
+ {
+ Readers[i] = DirectoryReader.Open(AddDir);
+ }
+ }
+
+ internal virtual void JoinThreads()
+ {
+ for (int i = 0; i < OuterInstance.NumThreads; i++)
+ {
+#if !NETSTANDARD
+ try
+ {
+#endif
+ Threads[i].Join();
+#if !NETSTANDARD
+ }
+ catch (ThreadInterruptedException ie)
+ {
+ throw new ThreadInterruptedException("Thread Interrupted Exception", ie);
+ }
+#endif
+ }
+ }
+
+ internal virtual void Close(bool doWait)
+ {
+ DidClose = true;
+ if (doWait)
+ {
+ MainWriter.WaitForMerges();
+ }
+ MainWriter.Dispose(doWait);
+ }
+
+ internal virtual void CloseDir()
+ {
+ for (int i = 0; i < NumDirs; i++)
+ {
+ Readers[i].Dispose();
+ }
+ AddDir.Dispose();
+ }
+
+ internal virtual void Handle(Exception t)
+ {
+ Console.WriteLine(t.StackTrace);
+ lock (Failures)
+ {
+ Failures.Add(t);
+ }
+ }
+
+ internal virtual void LaunchThreads(int numIter)
+ {
+ for (int i = 0; i < OuterInstance.NumThreads; i++)
+ {
+ Threads[i] = new ThreadAnonymousInnerClassHelper(this, numIter);
+ }
+ for (int i = 0; i < OuterInstance.NumThreads; i++)
+ {
+ Threads[i].Start();
+ }
+ }
+
+ private class ThreadAnonymousInnerClassHelper : ThreadClass
+ {
+ private readonly AddDirectoriesThreads OuterInstance;
+
+ private int NumIter;
+
+ public ThreadAnonymousInnerClassHelper(AddDirectoriesThreads outerInstance, int numIter)
+ {
+ this.OuterInstance = outerInstance;
+ this.NumIter = numIter;
+ }
+
+ public override void Run()
+ {
+ try
+ {
+ Directory[] dirs = new Directory[OuterInstance.NumDirs];
+ for (int k = 0; k < OuterInstance.NumDirs; k++)
+ {
+ dirs[k] = new MockDirectoryWrapper(Random(), new RAMDirectory(OuterInstance.AddDir, NewIOContext(Random())));
+ }
+ //int j = 0;
+ //while (true) {
+ // System.out.println(Thread.currentThread().getName() + ": iter
+ // j=" + j);
+ for (int x = 0; x < NumIter; x++)
+ {
+ // only do addIndexes
+ OuterInstance.DoBody(x, dirs);
+ }
+ //if (numIter > 0 && j == numIter)
+ // break;
+ //doBody(j++, dirs);
+ //doBody(5, dirs);
+ //}
+ }
+ catch (Exception t)
+ {
+ OuterInstance.Handle(t);
+ }
+ }
+ }
+
+ internal virtual void DoBody(int j, Directory[] dirs)
+ {
+ switch (j % 4)
+ {
+ case 0:
+ MainWriter.AddIndexes(dirs);
+ MainWriter.ForceMerge(1);
+ break;
+
+ case 1:
+ MainWriter.AddIndexes(dirs);
+ NumaddIndexes.IncrementAndGet();
+ break;
+
+ case 2:
+ MainWriter.AddIndexes(Readers);
+ break;
+
+ case 3:
+ MainWriter.Commit();
+ break;
+ }
+ Count.AddAndGet(dirs.Length * NUM_INIT_DOCS);
+ }
+ }
+
+ [Test]
+ public virtual void TestIndexWriterReopenSegmentFullMerge()
+ {
+ DoTestIndexWriterReopenSegment(true);
+ }
+
+ [Test]
+ public virtual void TestIndexWriterReopenSegment()
+ {
+ DoTestIndexWriterReopenSegment(false);
+ }
+
+ /// <summary>
+ /// Tests creating a segment, then check to insure the segment can be seen via
+ /// IW.getReader
+ /// </summary>
+ public virtual void DoTestIndexWriterReopenSegment(bool doFullMerge)
+ {
+ Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
+ IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ IndexReader r1 = writer.Reader;
+ Assert.AreEqual(0, r1.MaxDoc);
+ CreateIndexNoClose(false, "index1", writer);
+ writer.Flush(!doFullMerge, true);
+
+ IndexReader iwr1 = writer.Reader;
+ Assert.AreEqual(100, iwr1.MaxDoc);
+
+ IndexReader r2 = writer.Reader;
+ Assert.AreEqual(r2.MaxDoc, 100);
+ // add 100 documents
+ for (int x = 10000; x < 10000 + 100; x++)
+ {
+ Document d = DocHelper.CreateDocument(x, "index1", 5);
+ writer.AddDocument(d);
+ }
+ writer.Flush(false, true);
+ // verify the reader was reopened internally
+ IndexReader iwr2 = writer.Reader;
+ Assert.IsTrue(iwr2 != r1);
+ Assert.AreEqual(200, iwr2.MaxDoc);
+ // should have flushed out a segment
+ IndexReader r3 = writer.Reader;
+ Assert.IsTrue(r2 != r3);
+ Assert.AreEqual(200, r3.MaxDoc);
+
+ // dec ref the readers rather than close them because
+ // closing flushes changes to the writer
+ r1.Dispose();
+ iwr1.Dispose();
+ r2.Dispose();
+ r3.Dispose();
+ iwr2.Dispose();
+ writer.Dispose();
+
+ // test whether the changes made it to the directory
+ writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ IndexReader w2r1 = writer.Reader;
+ // insure the deletes were actually flushed to the directory
+ Assert.AreEqual(200, w2r1.MaxDoc);
+ w2r1.Dispose();
+ writer.Dispose();
+
+ dir1.Dispose();
+ }
+
+ /*
+ * Delete a document by term and return the doc id
+ *
+ * public static int deleteDocument(Term term, IndexWriter writer) throws
+ * IOException { IndexReader reader = writer.GetReader(); TermDocs td =
+ * reader.termDocs(term); int doc = -1; //if (td.Next()) { // doc = td.Doc();
+ * //} //writer.DeleteDocuments(term); td.Dispose(); return doc; }
+ */
+
+ public void CreateIndex(Random random, Directory dir1, string indexName, bool multiSegment)
+ {
+ IndexWriter w = new IndexWriter(dir1, NewIndexWriterConfig(random, TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetMergePolicy(new LogDocMergePolicy()));
+ for (int i = 0; i < 100; i++)
+ {
+ w.AddDocument(DocHelper.CreateDocument(i, indexName, 4));
+ }
+ if (!multiSegment)
+ {
+ w.ForceMerge(1);
+ }
+ w.Dispose();
+ }
+
+ public static void CreateIndexNoClose(bool multiSegment, string indexName, IndexWriter w)
+ {
+ for (int i = 0; i < 100; i++)
+ {
+ w.AddDocument(DocHelper.CreateDocument(i, indexName, 4));
+ }
+ if (!multiSegment)
+ {
+ w.ForceMerge(1);
+ }
+ }
+
+ private class MyWarmer : IndexWriter.IndexReaderWarmer
+ {
+ internal int WarmCount;
+
+ public override void Warm(AtomicReader reader)
+ {
+ WarmCount++;
+ }
+ }
+
+ [Test]
+ public virtual void TestMergeWarmer([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
+ {
+ Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
+ // Enroll warmer
+ MyWarmer warmer = new MyWarmer();
+ var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
+ .SetMaxBufferedDocs(2)
+ .SetMergedSegmentWarmer(warmer)
+ .SetMergeScheduler(scheduler)
+ .SetMergePolicy(NewLogMergePolicy());
+ IndexWriter writer = new IndexWriter(dir1, config);
+
+ // create the index
+ CreateIndexNoClose(false, "test", writer);
+
+ // get a reader to put writer into near real-time mode
+ IndexReader r1 = writer.Reader;
+
+ ((LogMergePolicy)writer.Config.MergePolicy).MergeFactor = 2;
+
+ //int num = AtLeast(100);
+ int num = 101;
+ for (int i = 0; i < num; i++)
+ {
+ writer.AddDocument(DocHelper.CreateDocument(i, "test", 4));
+ }
+ ((IConcurrentMergeScheduler)writer.Config.MergeScheduler).Sync();
+
+ Assert.IsTrue(warmer.WarmCount > 0);
+ Console.WriteLine("Count {0}", warmer.WarmCount);
+ int count = warmer.WarmCount;
+
+ var newDocument = DocHelper.CreateDocument(17, "test", 4);
+ writer.AddDocument(newDocument);
+ writer.ForceMerge(1);
+ Assert.IsTrue(warmer.WarmCount > count);
+
+ writer.Dispose();
+ r1.Dispose();
+ dir1.Dispose();
+ }
+
+ [Test]
+ public virtual void TestAfterCommit([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
+ {
+ Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
+ var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(scheduler);
+ IndexWriter writer = new IndexWriter(dir1, config);
+ writer.Commit();
+
+ // create the index
+ CreateIndexNoClose(false, "test", writer);
+
+ // get a reader to put writer into near real-time mode
+ DirectoryReader r1 = writer.Reader;
+ TestUtil.CheckIndex(dir1);
+ writer.Commit();
+ TestUtil.CheckIndex(dir1);
+ Assert.AreEqual(100, r1.NumDocs);
+
+ for (int i = 0; i < 10; i++)
+ {
+ writer.AddDocument(DocHelper.CreateDocument(i, "test", 4));
+ }
+ ((IConcurrentMergeScheduler)writer.Config.MergeScheduler).Sync();
+
+ DirectoryReader r2 = DirectoryReader.OpenIfChanged(r1);
+ if (r2 != null)
+ {
+ r1.Dispose();
+ r1 = r2;
+ }
+ Assert.AreEqual(110, r1.NumDocs);
+ writer.Dispose();
+ r1.Dispose();
+ dir1.Dispose();
+ }
+
+ // Make sure reader remains usable even if IndexWriter closes
+ [Test]
+ public virtual void TestAfterClose()
+ {
+ Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
+ IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ // create the index
+ CreateIndexNoClose(false, "test", writer);
+
+ DirectoryReader r = writer.Reader;
+ writer.Dispose();
+
+ TestUtil.CheckIndex(dir1);
+
+ // reader should remain usable even after IndexWriter is closed:
+ Assert.AreEqual(100, r.NumDocs);
+ Query q = new TermQuery(new Term("indexname", "test"));
+ IndexSearcher searcher = NewSearcher(r);
+ Assert.AreEqual(100, searcher.Search(q, 10).TotalHits);
+ try
+ {
+ DirectoryReader.OpenIfChanged(r);
+ Assert.Fail("failed to hit AlreadyClosedException");
+ }
+#pragma warning disable 168
+ catch (AlreadyClosedException ace)
+#pragma warning restore 168
+ {
+ // expected
+ }
+ r.Dispose();
+ dir1.Dispose();
+ }
+
+ // Stress test reopen during addIndexes
+ [Test]
+ public virtual void TestDuringAddIndexes()
+ {
+ Directory dir1 = GetAssertNoDeletesDirectory(NewDirectory());
+ IndexWriter writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy(2)));
+
+ // create the index
+ CreateIndexNoClose(false, "test", writer);
+ writer.Commit();
+
+ Directory[] dirs = new Directory[10];
+ for (int i = 0; i < 10; i++)
+ {
+ dirs[i] = new MockDirectoryWrapper(Random(), new RAMDirectory(dir1, NewIOContext(Random())));
+ }
+
+ DirectoryReader r = writer.Reader;
+
+ const float SECONDS = 0.5f;
+
+ long endTime = (long)(Environment.TickCount + 1000.0 * SECONDS);
+ IList<Exception> excs = new SynchronizedList<Exception>();
+
+ // Only one thread can addIndexes at a time, because
+ // IndexWriter acquires a write lock in each directory:
+ var threads = new ThreadClass[1];
+ for (int i = 0; i < threads.Length; i++)
+ {
+ threads[i] = new ThreadAnonymousInnerClassHelper(writer, dirs, endTime, excs);
+ threads[i].SetDaemon(true);
+ threads[i].Start();
+ }
+
+ int lastCount = 0;
+ while (Environment.TickCount < endTime)
+ {
+ DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
+ if (r2 != null)
+ {
+ r.Dispose();
+ r = r2;
+ }
+ Query q = new TermQuery(new Term("indexname", "test"));
+ IndexSearcher searcher = NewSearcher(r);
+ int count = searcher.Search(q, 10).TotalHits;
+ Assert.IsTrue(count >= lastCount);
+ lastCount = count;
+ }
+
+ for (int i = 0; i < threads.Length; i++)
+ {
+ threads[i].Join();
+ }
+ // final check
+ DirectoryReader dr2 = DirectoryReader.OpenIfChanged(r);
+ if (dr2 != null)
+ {
+ r.Dispose();
+ r = dr2;
+ }
+ Query q2 = new TermQuery(new Term("indexname", "test"));
+ IndexSearcher searcher_ = NewSearcher(r);
+ int count_ = searcher_.Search(q2, 10).TotalHits;
+ Assert.IsTrue(count_ >= lastCount);
+
+ Assert.AreEqual(0, excs.Count);
+ r.Dispose();
+ if (dir1 is MockDirectoryWrapper)
+ {
+ ICollection<string> openDeletedFiles = ((MockDirectoryWrapper)dir1).OpenDeletedFiles;
+ Assert.AreEqual(0, openDeletedFiles.Count, "openDeleted=" + openDeletedFiles);
+ }
+
+ writer.Dispose();
+
+ dir1.Dispose();
+ }
+
+ private class ThreadAnonymousInnerClassHelper : ThreadClass
+ {
+ private IndexWriter Writer;
+ private Directory[] Dirs;
+ private long EndTime;
+ private IList<Exception> Excs;
+
+ public ThreadAnonymousInnerClassHelper(IndexWriter writer, Directory[] dirs, long endTime, IList<Exception> excs)
+ {
+ this.Writer = writer;
+ this.Dirs = dirs;
+ this.EndTime = endTime;
+ this.Excs = excs;
+ }
+
+ public override void Run()
+ {
+ do
+ {
+ try
+ {
+ Writer.AddIndexes(Dirs);
+ Writer.MaybeMerge();
+ }
+ catch (Exception t)
+ {
+ Excs.Add(t);
+ throw new Exception(t.Message, t);
+ }
+ } while (Environment.TickCount < EndTime);
+ }
+ }
+
+ private Directory GetAssertNoDeletesDirectory(Directory directory)
+ {
+ if (directory is MockDirectoryWrapper)
+ {
+ ((MockDirectoryWrapper)directory).AssertNoDeleteOpenFile = true;
+ }
+ return directory;
+ }
+
+ // Stress test reopen during add/delete
+ [Test]
+ public virtual void TestDuringAddDelete()
+ {
+ Directory dir1 = NewDirectory();
+ var writer = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy(2)));
+
+ // create the index
+ CreateIndexNoClose(false, "test", writer);
+ writer.Commit();
+
+ DirectoryReader r = writer.Reader;
+
+ const float SECONDS = 0.5f;
+
+ long endTime = (long)(Environment.TickCount + 1000.0 * SECONDS);
+ ConcurrentQueue<Exception> excs = new ConcurrentQueue<Exception>();
+
+ var threads = new ThreadClass[NumThreads];
+ for (int i = 0; i < NumThreads; i++)
+ {
+ threads[i] = new ThreadAnonymousInnerClassHelper2(writer, r, endTime, excs);
+ threads[i].SetDaemon(true);
+ threads[i].Start();
+ }
+
+ int sum = 0;
+ while (Environment.TickCount < endTime)
+ {
+ DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
+ if (r2 != null)
+ {
+ r.Dispose();
+ r = r2;
+ }
+ Query q = new TermQuery(new Term("indexname", "test"));
+ IndexSearcher searcher = NewSearcher(r);
+ sum += searcher.Search(q, 10).TotalHits;
+ }
+
+ for (int i = 0; i < NumThreads; i++)
+ {
+ threads[i].Join();
+ }
+ // at least search once
+ DirectoryReader dr2 = DirectoryReader.OpenIfChanged(r);
+ if (dr2 != null)
+ {
+ r.Dispose();
+ r = dr2;
+ }
+ Query q2 = new TermQuery(new Term("indexname", "test"));
+ IndexSearcher indSearcher = NewSearcher(r);
+ sum += indSearcher.Search(q2, 10).TotalHits;
+ Assert.IsTrue(sum > 0, "no documents found at all");
+
+ Assert.AreEqual(0, excs.Count);
+ writer.Dispose();
+
+ r.Dispose();
+ dir1.Dispose();
+ }
+
+ private class ThreadAnonymousInnerClassHelper2 : ThreadClass
+ {
+ private IndexWriter Writer;
+ private DirectoryReader r;
+ private long EndTime;
+ private ConcurrentQueue<Exception> Excs;
+
+ public ThreadAnonymousInnerClassHelper2(IndexWriter writer, DirectoryReader r, long endTime, ConcurrentQueue<Exception> excs)
+ {
+ this.Writer = writer;
+ this.r = r;
+ this.EndTime = endTime;
+ this.Excs = excs;
+ rand = new Random(Random().Next());
+ }
+
+ internal readonly Random rand;
+
+ public override void Run()
+ {
+ int count = 0;
+ do
+ {
+ try
+ {
+ for (int docUpto = 0; docUpto < 10; docUpto++)
+ {
+ Writer.AddDocument(DocHelper.CreateDocument(10 * count + docUpto, "test", 4));
+ }
+ count++;
+ int limit = count * 10;
+ for (int delUpto = 0; delUpto < 5; delUpto++)
+ {
+ int x = rand.Next(limit);
+ Writer.DeleteDocuments(new Term("field3", "b" + x));
+ }
+ }
+ catch (Exception t)
+ {
+ Excs.Enqueue(t);
+ throw new Exception(t.Message, t);
+ }
+ } while (Environment.TickCount < EndTime);
+ }
+ }
+
+ [Test]
+ public virtual void TestForceMergeDeletes()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NewLogMergePolicy()));
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "a b c", Field.Store.NO));
+ Field id = NewStringField("id", "", Field.Store.NO);
+ doc.Add(id);
+ id.SetStringValue("0");
+ w.AddDocument(doc);
+ id.SetStringValue("1");
+ w.AddDocument(doc);
+ w.DeleteDocuments(new Term("id", "0"));
+
+ IndexReader r = w.Reader;
+ w.ForceMergeDeletes();
+ w.Dispose();
+ r.Dispose();
+ r = DirectoryReader.Open(dir);
+ Assert.AreEqual(1, r.NumDocs);
+ Assert.IsFalse(r.HasDeletions);
+ r.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestDeletesNumDocs()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "a b c", Field.Store.NO));
+ Field id = NewStringField("id", "", Field.Store.NO);
+ doc.Add(id);
+ id.SetStringValue("0");
+ w.AddDocument(doc);
+ id.SetStringValue("1");
+ w.AddDocument(doc);
+ IndexReader r = w.Reader;
+ Assert.AreEqual(2, r.NumDocs);
+ r.Dispose();
+
+ w.DeleteDocuments(new Term("id", "0"));
+ r = w.Reader;
+ Assert.AreEqual(1, r.NumDocs);
+ r.Dispose();
+
+ w.DeleteDocuments(new Term("id", "1"));
+ r = w.Reader;
+ Assert.AreEqual(0, r.NumDocs);
+ r.Dispose();
+
+ w.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestEmptyIndex()
+ {
+ // Ensures that getReader works on an empty index, which hasn't been committed yet.
+ Directory dir = NewDirectory();
+ IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ IndexReader r = w.Reader;
+ Assert.AreEqual(0, r.NumDocs);
+ r.Dispose();
+ w.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestSegmentWarmer()
+ {
+ Directory dir = NewDirectory();
+ AtomicBoolean didWarm = new AtomicBoolean();
+ IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetReaderPooling(true).SetMergedSegmentWarmer(new IndexReaderWarmerAnonymousInnerClassHelper(this, didWarm)).
+ SetMergePolicy(NewLogMergePolicy(10)));
+
+ Document doc = new Document();
+ doc.Add(NewStringField("foo", "bar", Field.Store.NO));
+ for (int i = 0; i < 20; i++)
+ {
+ w.AddDocument(doc);
+ }
+ w.WaitForMerges();
+ w.Dispose();
+ dir.Dispose();
+ Assert.IsTrue(didWarm.Get());
+ }
+
+ private class IndexReaderWarmerAnonymousInnerClassHelper : IndexWriter.IndexReaderWarmer
+ {
+ private readonly TestIndexWriterReader OuterInstance;
+
+ private AtomicBoolean DidWarm;
+
+ public IndexReaderWarmerAnonymousInnerClassHelper(TestIndexWriterReader outerInstance, AtomicBoolean didWarm)
+ {
+ this.OuterInstance = outerInstance;
+ this.DidWarm = didWarm;
+ }
+
+ public override void Warm(AtomicReader r)
+ {
+ IndexSearcher s = OuterInstance.NewSearcher(r);
+ TopDocs hits = s.Search(new TermQuery(new Term("foo", "bar")), 10);
+ Assert.AreEqual(20, hits.TotalHits);
+ DidWarm.Set(true);
+ }
+ }
+
+ [Test]
+ public virtual void TestSimpleMergedSegmentWramer()
+ {
+ Directory dir = NewDirectory();
+ AtomicBoolean didWarm = new AtomicBoolean();
+ InfoStream infoStream = new InfoStreamAnonymousInnerClassHelper(this, didWarm);
+ IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetReaderPooling(true).SetInfoStream(infoStream).SetMergedSegmentWarmer(new SimpleMergedSegmentWarmer(infoStream)).SetMergePolicy(NewLogMergePolicy(10)));
+
+ Document doc = new Document();
+ doc.Add(NewStringField("foo", "bar", Field.Store.NO));
+ for (int i = 0; i < 20; i++)
+ {
+ w.AddDocument(doc);
+ }
+ w.WaitForMerges();
+ w.Dispose();
+ dir.Dispose();
+ Assert.IsTrue(didWarm.Get());
+ }
+
+ private class InfoStreamAnonymousInnerClassHelper : InfoStream
+ {
+ private readonly TestIndexWriterReader OuterInstance;
+
+ private AtomicBoolean DidWarm;
+
+ public InfoStreamAnonymousInnerClassHelper(TestIndexWriterReader outerInstance, AtomicBoolean didWarm)
+ {
+ this.OuterInstance = outerInstance;
+ this.DidWarm = didWarm;
+ }
+
+ public override void Dispose()
+ {
+ }
+
+ public override void Message(string component, string message)
+ {
+ if ("SMSW".Equals(component))
+ {
+ DidWarm.Set(true);
+ }
+ }
+
+ public override bool IsEnabled(string component)
+ {
+ return true;
+ }
+ }
+
+ [Test]
+ public virtual void TestNoTermsIndex()
+ {
+ // Some Codecs don't honor the ReaderTermsIndexDivisor, so skip the test if
+ // they're picked.
+ AssumeFalse("PreFlex codec does not support ReaderTermsIndexDivisor!", "Lucene3x".Equals(Codec.Default.Name));
+
+ IndexWriterConfig conf = (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetReaderTermsIndexDivisor(-1);
+
+ // Don't proceed if picked Codec is in the list of illegal ones.
+ string format = TestUtil.GetPostingsFormat("f");
+ AssumeFalse("Format: " + format + " does not support ReaderTermsIndexDivisor!", (format.Equals("FSTPulsing41") || format.Equals("FSTOrdPulsing41") || format.Equals("FST41") || format.Equals("FSTOrd41") || format.Equals("SimpleText") || format.Equals("Memory") || format.Equals("MockRandom") || format.Equals("Direct")));
+
+ Directory dir = NewDirectory();
+ IndexWriter w = new IndexWriter(dir, conf);
+ Document doc = new Document();
+ doc.Add(new TextField("f", "val", Field.Store.NO));
+ w.AddDocument(doc);
+ SegmentReader r = GetOnlySegmentReader(DirectoryReader.Open(w, true));
+ try
+ {
+ TestUtil.Docs(Random(), r, "f", new BytesRef("val"), null, null, DocsEnum.FLAG_NONE);
+ Assert.Fail("should have failed to seek since terms index was not loaded.");
+ }
+#pragma warning disable 168
+ catch (InvalidOperationException e)
+#pragma warning restore 168
+ {
+ // expected - we didn't load the term index
+ }
+ finally
+ {
+ r.Dispose();
+ w.Dispose();
+ dir.Dispose();
+ }
+ }
+
+ [Test]
+ public virtual void TestReopenAfterNoRealChange()
+ {
+ Directory d = GetAssertNoDeletesDirectory(NewDirectory());
+ IndexWriter w = new IndexWriter(d, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ DirectoryReader r = w.Reader; // start pooling readers
+
+ DirectoryReader r2 = DirectoryReader.OpenIfChanged(r);
+ Assert.IsNull(r2);
+
+ w.AddDocument(new Document());
+ DirectoryReader r3 = DirectoryReader.OpenIfChanged(r);
+ Assert.IsNotNull(r3);
+ Assert.IsTrue(r3.Version != r.Version);
+ Assert.IsTrue(r3.IsCurrent);
+
+ // Deletes nothing in reality...:
+ w.DeleteDocuments(new Term("foo", "bar"));
+
+ // ... but IW marks this as not current:
+ Assert.IsFalse(r3.IsCurrent);
+ DirectoryReader r4 = DirectoryReader.OpenIfChanged(r3);
+ Assert.IsNull(r4);
+
+ // Deletes nothing in reality...:
+ w.DeleteDocuments(new Term("foo", "bar"));
+ DirectoryReader r5 = DirectoryReader.OpenIfChanged(r3, w, true);
+ Assert.IsNull(r5);
+
+ r3.Dispose();
+
+ w.Dispose();
+ d.Dispose();
+ }
+
+ [Test]
+ public virtual void TestNRTOpenExceptions()
+ {
+ // LUCENE-5262: test that several failed attempts to obtain an NRT reader
+ // don't leak file handles.
+ MockDirectoryWrapper dir = (MockDirectoryWrapper)GetAssertNoDeletesDirectory(NewMockDirectory());
+ AtomicBoolean shouldFail = new AtomicBoolean();
+ dir.FailOn(new FailureAnonymousInnerClassHelper(shouldFail));
+
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMergePolicy(NoMergePolicy.COMPOUND_FILES); // prevent merges from getting in the way
+ IndexWriter writer = new IndexWriter(dir, conf);
+
+ // create a segment and open an NRT reader
+ writer.AddDocument(new Document());
+ writer.Reader.Dispose();
+
+ // add a new document so a new NRT reader is required
+ writer.AddDocument(new Document());
+
+ // try to obtain an NRT reader twice: first time it fails and closes all the
+ // other NRT readers. second time it fails, but also fails to close the
+ // other NRT reader, since it is already marked closed!
+ for (int i = 0; i < 2; i++)
+ {
+ shouldFail.Set(true);
+ try
+ {
+ writer.Reader.Dispose();
+ }
+#pragma warning disable 168
+ catch (FakeIOException e)
+#pragma warning restore 168
+ {
+ // expected
+ if (VERBOSE)
+ {
+ Console.WriteLine("hit expected fake IOE");
+ }
+ }
+ }
+
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ private class FailureAnonymousInnerClassHelper : MockDirectoryWrapper.Failure
+ {
+ private readonly AtomicBoolean ShouldFail;
+
+ public FailureAnonymousInnerClassHelper(AtomicBoolean shouldFail)
+ {
+ this.ShouldFail = shouldFail;
+ }
+
+ public override void Eval(MockDirectoryWrapper dir)
+ {
+ if (ShouldFail.Get() && StackTraceHelper.DoesStackTraceContainMethod("GetReadOnlyClone"))
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: now fail; exc:");
+ Console.WriteLine((new Exception()).StackTrace);
+ }
+ ShouldFail.Set(false);
+ throw new FakeIOException();
+ }
+ }
+ }
+
+ /// <summary>
+ /// Make sure if all we do is open NRT reader against
+ /// writer, we don't see merge starvation.
+ /// </summary>
+ [Test]
+ public virtual void TestTooManySegments()
+ {
+ Directory dir = GetAssertNoDeletesDirectory(NewDirectory());
+ // Don't use newIndexWriterConfig, because we need a
+ // "sane" mergePolicy:
+ IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ IndexWriter w = new IndexWriter(dir, iwc);
+ // Create 500 segments:
+ for (int i = 0; i < 500; i++)
+ {
+ Document doc = new Document();
+ doc.Add(NewStringField("id", "" + i, Field.Store.NO));
+ w.AddDocument(doc);
+ IndexReader r = DirectoryReader.Open(w, true);
+ // Make sure segment count never exceeds 100:
+ Assert.IsTrue(r.Leaves.Count < 100);
+ r.Dispose();
+ }
+ w.Dispose();
+ dir.Dispose();
+ }
+ }
+}
\ No newline at end of file