You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/02/26 23:37:20 UTC
[32/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests:
Removed \core directory and put its contents in root directory
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestIndexWriter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestIndexWriter.cs b/src/Lucene.Net.Tests/Index/TestIndexWriter.cs
new file mode 100644
index 0000000..1c3a56a
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestIndexWriter.cs
@@ -0,0 +1,2888 @@
+using Lucene.Net.Analysis.TokenAttributes;
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Threading;
+using Lucene.Net.Documents;
+using Lucene.Net.Search;
+
+namespace Lucene.Net.Index
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using Lucene.Net.Analysis;
+ using Lucene.Net.Randomized.Generators;
+ using Lucene.Net.Support;
+ using Lucene.Net.Util;
+ using NUnit.Framework;
+ using System.Diagnostics;
+ using System.IO;
+ using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
+ using Automaton = Lucene.Net.Util.Automaton.Automaton;
+ using BaseDirectoryWrapper = Lucene.Net.Store.BaseDirectoryWrapper;
+ using BasicAutomata = Lucene.Net.Util.Automaton.BasicAutomata;
+
+ //using SimpleTextCodec = Lucene.Net.Codecs.simpletext.SimpleTextCodec;
+ using BinaryDocValuesField = BinaryDocValuesField;
+ using IBits = Lucene.Net.Util.IBits;
+ using BytesRef = Lucene.Net.Util.BytesRef;
+ using CharacterRunAutomaton = Lucene.Net.Util.Automaton.CharacterRunAutomaton;
+ using CharTermAttribute = Lucene.Net.Analysis.TokenAttributes.CharTermAttribute;
+ using Constants = Lucene.Net.Util.Constants;
+ using Directory = Lucene.Net.Store.Directory;
+ using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator;
+ using Document = Documents.Document;
+ using Field = Field;
+ using FieldType = FieldType;
+ using IndexOutput = Lucene.Net.Store.IndexOutput;
+ using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+ using IOContext = Lucene.Net.Store.IOContext;
+ using IOUtils = Lucene.Net.Util.IOUtils;
+ using Lock = Lucene.Net.Store.Lock;
+ using LockFactory = Lucene.Net.Store.LockFactory;
+ using LockObtainFailedException = Lucene.Net.Store.LockObtainFailedException;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+ using MatchAllDocsQuery = Lucene.Net.Search.MatchAllDocsQuery;
+ using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+ using NoLockFactory = Lucene.Net.Store.NoLockFactory;
+ using NumericDocValuesField = NumericDocValuesField;
+ using PackedInt32s = Lucene.Net.Util.Packed.PackedInt32s;
+ using PhraseQuery = Lucene.Net.Search.PhraseQuery;
+ using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+ using ScoreDoc = Lucene.Net.Search.ScoreDoc;
+ using SimpleFSLockFactory = Lucene.Net.Store.SimpleFSLockFactory;
+ using SingleInstanceLockFactory = Lucene.Net.Store.SingleInstanceLockFactory;
+ using SortedDocValuesField = SortedDocValuesField;
+ using SortedSetDocValuesField = SortedSetDocValuesField;
+ using StoredField = StoredField;
+ using StringField = StringField;
+ using TermQuery = Lucene.Net.Search.TermQuery;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+ using TextField = TextField;
+
+ [TestFixture]
+ public class TestIndexWriter : LuceneTestCase
+ {
+ private static readonly FieldType StoredTextType = new FieldType(TextField.TYPE_NOT_STORED);
+
+ [Test]
+ public virtual void TestDocCount()
+ {
+ Directory dir = NewDirectory();
+
+ IndexWriter writer = null;
+ IndexReader reader = null;
+ int i;
+
+ long savedWriteLockTimeout = IndexWriterConfig.DefaultWriteLockTimeout;
+ try
+ {
+ IndexWriterConfig.DefaultWriteLockTimeout = 2000;
+ Assert.AreEqual(2000, IndexWriterConfig.DefaultWriteLockTimeout);
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ }
+ finally
+ {
+ IndexWriterConfig.DefaultWriteLockTimeout = savedWriteLockTimeout;
+ }
+
+ // add 100 documents
+ for (i = 0; i < 100; i++)
+ {
+ AddDocWithIndex(writer, i);
+ }
+ Assert.AreEqual(100, writer.MaxDoc);
+ writer.Dispose();
+
+ // delete 40 documents
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES));
+ for (i = 0; i < 40; i++)
+ {
+ writer.DeleteDocuments(new Term("id", "" + i));
+ }
+ writer.Dispose();
+
+ reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(60, reader.NumDocs);
+ reader.Dispose();
+
+ // merge the index down and check that the new doc count is correct
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Assert.AreEqual(60, writer.NumDocs);
+ writer.ForceMerge(1);
+ Assert.AreEqual(60, writer.MaxDoc);
+ Assert.AreEqual(60, writer.NumDocs);
+ writer.Dispose();
+
+ // check that the index reader gives the same numbers.
+ reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(60, reader.MaxDoc);
+ Assert.AreEqual(60, reader.NumDocs);
+ reader.Dispose();
+
+ // make sure opening a new index for create over
+ // this existing one works correctly:
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.CREATE));
+ Assert.AreEqual(0, writer.MaxDoc);
+ Assert.AreEqual(0, writer.NumDocs);
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ /// <summary>
+ /// LUCENENET specific
+ /// Changed from internal static method to private to remove
+ /// inter-dependencies between TestIndexWriter*.cs, TestAddIndexes.cs
+ /// and TestDeletionPolicy.cs tests
+ /// </summary>
+ private void AddDoc(IndexWriter writer)
+ {
+ Document doc = new Document();
+ doc.Add(NewTextField("content", "aaa", Field.Store.NO));
+ writer.AddDocument(doc);
+ }
+
+ /// <summary>
+ /// LUCENENET specific
+ /// Changed from internal static method to private to remove
+ /// inter-dependencies between TestIndexWriter*.cs, TestAddIndexes.cs
+ /// and TestDeletionPolicy.cs tests
+ /// </summary>
+ private void AddDocWithIndex(IndexWriter writer, int index)
+ {
+ Document doc = new Document();
+ doc.Add(NewField("content", "aaa " + index, StoredTextType));
+ doc.Add(NewField("id", "" + index, StoredTextType));
+ writer.AddDocument(doc);
+ }
+
+ public static void AssertNoUnreferencedFiles(Directory dir, string message)
+ {
+ string[] startFiles = dir.ListAll();
+ (new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())))).Rollback();
+ string[] endFiles = dir.ListAll();
+
+ Array.Sort(startFiles);
+ Array.Sort(endFiles);
+
+ if (!Arrays.Equals(startFiles, endFiles))
+ {
+ Assert.Fail(message + ": before delete:\n " + ArrayToString(startFiles) + "\n after delete:\n " + ArrayToString(endFiles));
+ }
+ }
+
+ internal static string ArrayToString(string[] l)
+ {
+ string s = "";
+ for (int i = 0; i < l.Length; i++)
+ {
+ if (i > 0)
+ {
+ s += "\n ";
+ }
+ s += l[i];
+ }
+ return s;
+ }
+
+ // Make sure we can open an index for create even when a
+ // reader holds it open (this fails pre lock-less
+ // commits on windows):
+ [Test]
+ public virtual void TestCreateWithReader()
+ {
+ Directory dir = NewDirectory();
+
+ // add one document & close writer
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ AddDoc(writer);
+ writer.Dispose();
+
+ // now open reader:
+ IndexReader reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(reader.NumDocs, 1, "should be one document");
+
+ // now open index for create:
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.CREATE));
+ Assert.AreEqual(writer.MaxDoc, 0, "should be zero documents");
+ AddDoc(writer);
+ writer.Dispose();
+
+ Assert.AreEqual(reader.NumDocs, 1, "should be one document");
+ IndexReader reader2 = DirectoryReader.Open(dir);
+ Assert.AreEqual(reader2.NumDocs, 1, "should be one document");
+ reader.Dispose();
+ reader2.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestChangesAfterClose([ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler)
+ {
+ Directory dir = NewDirectory();
+
+ IndexWriter writer = null;
+
+ var config = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergeScheduler(scheduler);
+ writer = new IndexWriter(dir, config);
+ AddDoc(writer);
+
+ // close
+ writer.Dispose();
+ try
+ {
+ AddDoc(writer);
+ Assert.Fail("did not hit AlreadyClosedException");
+ }
+#pragma warning disable 168
+ catch (AlreadyClosedException e)
+#pragma warning restore 168
+ {
+ // expected
+ }
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestIndexNoDocuments()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ writer.Commit();
+ writer.Dispose();
+
+ IndexReader reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(0, reader.MaxDoc);
+ Assert.AreEqual(0, reader.NumDocs);
+ reader.Dispose();
+
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND));
+ writer.Commit();
+ writer.Dispose();
+
+ reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(0, reader.MaxDoc);
+ Assert.AreEqual(0, reader.NumDocs);
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestManyFields()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(10));
+ for (int j = 0; j < 100; j++)
+ {
+ Document doc = new Document();
+ doc.Add(NewField("a" + j, "aaa" + j, StoredTextType));
+ doc.Add(NewField("b" + j, "aaa" + j, StoredTextType));
+ doc.Add(NewField("c" + j, "aaa" + j, StoredTextType));
+ doc.Add(NewField("d" + j, "aaa", StoredTextType));
+ doc.Add(NewField("e" + j, "aaa", StoredTextType));
+ doc.Add(NewField("f" + j, "aaa", StoredTextType));
+ writer.AddDocument(doc);
+ }
+ writer.Dispose();
+
+ IndexReader reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(100, reader.MaxDoc);
+ Assert.AreEqual(100, reader.NumDocs);
+ for (int j = 0; j < 100; j++)
+ {
+ Assert.AreEqual(1, reader.DocFreq(new Term("a" + j, "aaa" + j)));
+ Assert.AreEqual(1, reader.DocFreq(new Term("b" + j, "aaa" + j)));
+ Assert.AreEqual(1, reader.DocFreq(new Term("c" + j, "aaa" + j)));
+ Assert.AreEqual(1, reader.DocFreq(new Term("d" + j, "aaa")));
+ Assert.AreEqual(1, reader.DocFreq(new Term("e" + j, "aaa")));
+ Assert.AreEqual(1, reader.DocFreq(new Term("f" + j, "aaa")));
+ }
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestSmallRAMBuffer()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetRAMBufferSizeMB(0.000001).SetMergePolicy(NewLogMergePolicy(10)));
+ int lastNumFile = dir.ListAll().Length;
+ for (int j = 0; j < 9; j++)
+ {
+ Document doc = new Document();
+ doc.Add(NewField("field", "aaa" + j, StoredTextType));
+ writer.AddDocument(doc);
+ int numFile = dir.ListAll().Length;
+ // Verify that with a tiny RAM buffer we see new
+ // segment after every doc
+ Assert.IsTrue(numFile > lastNumFile);
+ lastNumFile = numFile;
+ }
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ // Make sure it's OK to change RAM buffer size and
+ // maxBufferedDocs in a write session
+ [Test]
+ public virtual void TestChangingRAMBuffer()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ writer.Config.SetMaxBufferedDocs(10);
+ writer.Config.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+
+ int lastFlushCount = -1;
+ for (int j = 1; j < 52; j++)
+ {
+ Document doc = new Document();
+ doc.Add(new Field("field", "aaa" + j, StoredTextType));
+ writer.AddDocument(doc);
+ TestUtil.SyncConcurrentMerges(writer);
+ int flushCount = writer.FlushCount;
+ if (j == 1)
+ {
+ lastFlushCount = flushCount;
+ }
+ else if (j < 10)
+ // No new files should be created
+ {
+ Assert.AreEqual(flushCount, lastFlushCount);
+ }
+ else if (10 == j)
+ {
+ Assert.IsTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ writer.Config.SetRAMBufferSizeMB(0.000001);
+ writer.Config.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ }
+ else if (j < 20)
+ {
+ Assert.IsTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ }
+ else if (20 == j)
+ {
+ writer.Config.SetRAMBufferSizeMB(16);
+ writer.Config.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ }
+ else if (j < 30)
+ {
+ Assert.AreEqual(flushCount, lastFlushCount);
+ }
+ else if (30 == j)
+ {
+ writer.Config.SetRAMBufferSizeMB(0.000001);
+ writer.Config.SetMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ }
+ else if (j < 40)
+ {
+ Assert.IsTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ }
+ else if (40 == j)
+ {
+ writer.Config.SetMaxBufferedDocs(10);
+ writer.Config.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ }
+ else if (j < 50)
+ {
+ Assert.AreEqual(flushCount, lastFlushCount);
+ writer.Config.SetMaxBufferedDocs(10);
+ writer.Config.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ }
+ else if (50 == j)
+ {
+ Assert.IsTrue(flushCount > lastFlushCount);
+ }
+ }
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestChangingRAMBuffer2()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ writer.Config.SetMaxBufferedDocs(10);
+ writer.Config.SetMaxBufferedDeleteTerms(10);
+ writer.Config.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+
+ for (int j = 1; j < 52; j++)
+ {
+ Document doc = new Document();
+ doc.Add(new Field("field", "aaa" + j, StoredTextType));
+ writer.AddDocument(doc);
+ }
+
+ int lastFlushCount = -1;
+ for (int j = 1; j < 52; j++)
+ {
+ writer.DeleteDocuments(new Term("field", "aaa" + j));
+ TestUtil.SyncConcurrentMerges(writer);
+ int flushCount = writer.FlushCount;
+
+ if (j == 1)
+ {
+ lastFlushCount = flushCount;
+ }
+ else if (j < 10)
+ {
+ // No new files should be created
+ Assert.AreEqual(flushCount, lastFlushCount);
+ }
+ else if (10 == j)
+ {
+ Assert.IsTrue(flushCount > lastFlushCount, "" + j);
+ lastFlushCount = flushCount;
+ writer.Config.SetRAMBufferSizeMB(0.000001);
+ writer.Config.SetMaxBufferedDeleteTerms(1);
+ }
+ else if (j < 20)
+ {
+ Assert.IsTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ }
+ else if (20 == j)
+ {
+ writer.Config.SetRAMBufferSizeMB(16);
+ writer.Config.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ }
+ else if (j < 30)
+ {
+ Assert.AreEqual(flushCount, lastFlushCount);
+ }
+ else if (30 == j)
+ {
+ writer.Config.SetRAMBufferSizeMB(0.000001);
+ writer.Config.SetMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ writer.Config.SetMaxBufferedDeleteTerms(1);
+ }
+ else if (j < 40)
+ {
+ Assert.IsTrue(flushCount > lastFlushCount);
+ lastFlushCount = flushCount;
+ }
+ else if (40 == j)
+ {
+ writer.Config.SetMaxBufferedDeleteTerms(10);
+ writer.Config.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ lastFlushCount = flushCount;
+ }
+ else if (j < 50)
+ {
+ Assert.AreEqual(flushCount, lastFlushCount);
+ writer.Config.SetMaxBufferedDeleteTerms(10);
+ writer.Config.SetRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
+ }
+ else if (50 == j)
+ {
+ Assert.IsTrue(flushCount > lastFlushCount);
+ }
+ }
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestDiverseDocs()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetRAMBufferSizeMB(0.5));
+ int n = AtLeast(1);
+ for (int i = 0; i < n; i++)
+ {
+ // First, docs where every term is unique (heavy on
+ // Posting instances)
+ for (int j = 0; j < 100; j++)
+ {
+ Document doc = new Document();
+ for (int k = 0; k < 100; k++)
+ {
+ doc.Add(NewField("field", Convert.ToString(Random().Next()), StoredTextType));
+ }
+ writer.AddDocument(doc);
+ }
+
+ // Next, many single term docs where only one term
+ // occurs (heavy on byte blocks)
+ for (int j = 0; j < 100; j++)
+ {
+ Document doc = new Document();
+ doc.Add(NewField("field", "aaa aaa aaa aaa aaa aaa aaa aaa aaa aaa", StoredTextType));
+ writer.AddDocument(doc);
+ }
+
+ // Next, many single term docs where only one term
+ // occurs but the terms are very long (heavy on
+ // char[] arrays)
+ for (int j = 0; j < 100; j++)
+ {
+ StringBuilder b = new StringBuilder();
+ string x = Convert.ToString(j) + ".";
+ for (int k = 0; k < 1000; k++)
+ {
+ b.Append(x);
+ }
+ string longTerm = b.ToString();
+
+ Document doc = new Document();
+ doc.Add(NewField("field", longTerm, StoredTextType));
+ writer.AddDocument(doc);
+ }
+ }
+ writer.Dispose();
+
+ IndexReader reader = DirectoryReader.Open(dir);
+ IndexSearcher searcher = NewSearcher(reader);
+ int totalHits = searcher.Search(new TermQuery(new Term("field", "aaa")), null, 1).TotalHits;
+ Assert.AreEqual(n * 100, totalHits);
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestEnablingNorms()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(10));
+ // Enable norms for only 1 doc, pre flush
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.OmitNorms = true;
+ for (int j = 0; j < 10; j++)
+ {
+ Document doc = new Document();
+ Field f = null;
+ if (j != 8)
+ {
+ f = NewField("field", "aaa", customType);
+ }
+ else
+ {
+ f = NewField("field", "aaa", StoredTextType);
+ }
+ doc.Add(f);
+ writer.AddDocument(doc);
+ }
+ writer.Dispose();
+
+ Term searchTerm = new Term("field", "aaa");
+
+ IndexReader reader = DirectoryReader.Open(dir);
+ IndexSearcher searcher = NewSearcher(reader);
+ ScoreDoc[] hits = searcher.Search(new TermQuery(searchTerm), null, 1000).ScoreDocs;
+ Assert.AreEqual(10, hits.Length);
+ reader.Dispose();
+
+ writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.CREATE).SetMaxBufferedDocs(10));
+ // Enable norms for only 1 doc, post flush
+ for (int j = 0; j < 27; j++)
+ {
+ Document doc = new Document();
+ Field f = null;
+ if (j != 26)
+ {
+ f = NewField("field", "aaa", customType);
+ }
+ else
+ {
+ f = NewField("field", "aaa", StoredTextType);
+ }
+ doc.Add(f);
+ writer.AddDocument(doc);
+ }
+ writer.Dispose();
+ reader = DirectoryReader.Open(dir);
+ searcher = NewSearcher(reader);
+ hits = searcher.Search(new TermQuery(searchTerm), null, 1000).ScoreDocs;
+ Assert.AreEqual(27, hits.Length);
+ reader.Dispose();
+
+ reader = DirectoryReader.Open(dir);
+ reader.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestHighFreqTerm()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetRAMBufferSizeMB(0.01));
+ // Massive doc that has 128 K a's
+ StringBuilder b = new StringBuilder(1024 * 1024);
+ for (int i = 0; i < 4096; i++)
+ {
+ b.Append(" a a a a a a a a");
+ b.Append(" a a a a a a a a");
+ b.Append(" a a a a a a a a");
+ b.Append(" a a a a a a a a");
+ }
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.StoreTermVectors = true;
+ customType.StoreTermVectorPositions = true;
+ customType.StoreTermVectorOffsets = true;
+ doc.Add(NewField("field", b.ToString(), customType));
+ writer.AddDocument(doc);
+ writer.Dispose();
+
+ IndexReader reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(1, reader.MaxDoc);
+ Assert.AreEqual(1, reader.NumDocs);
+ Term t = new Term("field", "a");
+ Assert.AreEqual(1, reader.DocFreq(t));
+ DocsEnum td = TestUtil.Docs(Random(), reader, "field", new BytesRef("a"), MultiFields.GetLiveDocs(reader), null, DocsEnum.FLAG_FREQS);
+ td.NextDoc();
+ Assert.AreEqual(128 * 1024, td.Freq);
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ //Helper class for TestNullLockFactory
+ public class MyRAMDirectory : MockDirectoryWrapper
+ {
+ private LockFactory myLockFactory;
+
+ public MyRAMDirectory(Directory @delegate)
+ : base(Random(), @delegate)
+ {
+ LockFactory_Renamed = null;
+ myLockFactory = new SingleInstanceLockFactory();
+ }
+
+ public override Lock MakeLock(string name)
+ {
+ return myLockFactory.MakeLock(name);
+ }
+ }
+
+ // Make sure that a Directory implementation that does
+ // not use LockFactory at all (ie overrides makeLock and
+ // implements its own private locking) works OK. this
+ // was raised on java-dev as loss of backwards
+ // compatibility.
+ [Test]
+ public virtual void TestNullLockFactory()
+ {
+ Directory dir = new MyRAMDirectory(new RAMDirectory());
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ for (int i = 0; i < 100; i++)
+ {
+ AddDoc(writer);
+ }
+ writer.Dispose();
+ Term searchTerm = new Term("content", "aaa");
+ IndexReader reader = DirectoryReader.Open(dir);
+ IndexSearcher searcher = NewSearcher(reader);
+ ScoreDoc[] hits = searcher.Search(new TermQuery(searchTerm), null, 1000).ScoreDocs;
+ Assert.AreEqual(100, hits.Length, "did not get right number of hits");
+ reader.Dispose();
+
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.CREATE));
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestFlushWithNoMerging()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy(10)));
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.StoreTermVectors = true;
+ customType.StoreTermVectorPositions = true;
+ customType.StoreTermVectorOffsets = true;
+ doc.Add(NewField("field", "aaa", customType));
+ for (int i = 0; i < 19; i++)
+ {
+ writer.AddDocument(doc);
+ }
+ writer.Flush(false, true);
+ writer.Dispose();
+ SegmentInfos sis = new SegmentInfos();
+ sis.Read(dir);
+ // Since we flushed w/o allowing merging we should now
+ // have 10 segments
+ Assert.AreEqual(10, sis.Count);
+ dir.Dispose();
+ }
+
+ // Make sure we can flush segment w/ norms, then add
+ // empty doc (no norms) and flush
+ [Test]
+ public virtual void TestEmptyDocAfterFlushingRealDoc()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.StoreTermVectors = true;
+ customType.StoreTermVectorPositions = true;
+ customType.StoreTermVectorOffsets = true;
+ doc.Add(NewField("field", "aaa", customType));
+ writer.AddDocument(doc);
+ writer.Commit();
+ if (VERBOSE)
+ {
+ Console.WriteLine("\nTEST: now add empty doc");
+ }
+ writer.AddDocument(new Document());
+ writer.Dispose();
+ IndexReader reader = DirectoryReader.Open(dir);
+ Assert.AreEqual(2, reader.NumDocs);
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ /// <summary>
+ /// Test that no NullPointerException will be raised,
+ /// when adding one document with a single, empty field
+ /// and term vectors enabled.
+ /// </summary>
+ [Test]
+ public virtual void TestBadSegment()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter iw = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ Document document = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.StoreTermVectors = true;
+ document.Add(NewField("tvtest", "", customType));
+ iw.AddDocument(document);
+ iw.Dispose();
+ dir.Dispose();
+ }
+
+#if !NETSTANDARD //NOTE: Cannot set ThreadPriority in .NET Core.
+ // LUCENE-1036
+ [Test]
+ public virtual void TestMaxThreadPriority()
+ {
+ ThreadPriority pri = ThreadClass.Current().Priority;
+ try
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig conf = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy());
+ ((LogMergePolicy)conf.MergePolicy).MergeFactor = 2;
+ IndexWriter iw = new IndexWriter(dir, conf);
+ Document document = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.StoreTermVectors = true;
+ document.Add(NewField("tvtest", "a b c", customType));
+ Thread.CurrentThread.Priority = ThreadPriority.Highest;
+ for (int i = 0; i < 4; i++)
+ {
+ iw.AddDocument(document);
+ }
+ iw.Dispose();
+ dir.Dispose();
+ }
+ finally
+ {
+ Thread.CurrentThread.Priority = pri;
+ }
+ }
+#endif
+
+ [Test]
+ public virtual void TestVariableSchema()
+ {
+ Directory dir = NewDirectory();
+ for (int i = 0; i < 20; i++)
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: iter=" + i);
+ }
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy()));
+ //LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
+ //lmp.setMergeFactor(2);
+ //lmp.setNoCFSRatio(0.0);
+ Document doc = new Document();
+ string contents = "aa bb cc dd ee ff gg hh ii jj kk";
+
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ FieldType type = null;
+ if (i == 7)
+ {
+ // Add empty docs here
+ doc.Add(NewTextField("content3", "", Field.Store.NO));
+ }
+ else
+ {
+ if (i % 2 == 0)
+ {
+ doc.Add(NewField("content4", contents, customType));
+ type = customType;
+ }
+ else
+ {
+ type = TextField.TYPE_NOT_STORED;
+ }
+ doc.Add(NewTextField("content1", contents, Field.Store.NO));
+ doc.Add(NewField("content3", "", customType));
+ doc.Add(NewField("content5", "", type));
+ }
+
+ for (int j = 0; j < 4; j++)
+ {
+ writer.AddDocument(doc);
+ }
+
+ writer.Dispose();
+
+ if (0 == i % 4)
+ {
+ writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ //LogMergePolicy lmp2 = (LogMergePolicy) writer.getConfig().getMergePolicy();
+ //lmp2.setNoCFSRatio(0.0);
+ writer.ForceMerge(1);
+ writer.Dispose();
+ }
+ }
+ dir.Dispose();
+ }
+
+ // LUCENE-1084: test unlimited field length
+ [Test]
+ public virtual void TestUnlimitedMaxFieldLength()
+ {
+ Directory dir = NewDirectory();
+
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ Document doc = new Document();
+ StringBuilder b = new StringBuilder();
+ for (int i = 0; i < 10000; i++)
+ {
+ b.Append(" a");
+ }
+ b.Append(" x");
+ doc.Add(NewTextField("field", b.ToString(), Field.Store.NO));
+ writer.AddDocument(doc);
+ writer.Dispose();
+
+ IndexReader reader = DirectoryReader.Open(dir);
+ Term t = new Term("field", "x");
+ Assert.AreEqual(1, reader.DocFreq(t));
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ // LUCENE-1179
+ [Test]
+ public virtual void TestEmptyFieldName()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ doc.Add(NewTextField("", "a b c", Field.Store.NO));
+ writer.AddDocument(doc);
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestEmptyFieldNameTerms()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ doc.Add(NewTextField("", "a b c", Field.Store.NO));
+ writer.AddDocument(doc);
+ writer.Dispose();
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader subreader = GetOnlySegmentReader(reader);
+ TermsEnum te = subreader.Fields.GetTerms("").GetIterator(null);
+ Assert.AreEqual(new BytesRef("a"), te.Next());
+ Assert.AreEqual(new BytesRef("b"), te.Next());
+ Assert.AreEqual(new BytesRef("c"), te.Next());
+ Assert.IsNull(te.Next());
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestEmptyFieldNameWithEmptyTerm()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ doc.Add(NewStringField("", "", Field.Store.NO));
+ doc.Add(NewStringField("", "a", Field.Store.NO));
+ doc.Add(NewStringField("", "b", Field.Store.NO));
+ doc.Add(NewStringField("", "c", Field.Store.NO));
+ writer.AddDocument(doc);
+ writer.Dispose();
+ DirectoryReader reader = DirectoryReader.Open(dir);
+ AtomicReader subreader = GetOnlySegmentReader(reader);
+ TermsEnum te = subreader.Fields.GetTerms("").GetIterator(null);
+ Assert.AreEqual(new BytesRef(""), te.Next());
+ Assert.AreEqual(new BytesRef("a"), te.Next());
+ Assert.AreEqual(new BytesRef("b"), te.Next());
+ Assert.AreEqual(new BytesRef("c"), te.Next());
+ Assert.IsNull(te.Next());
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ private sealed class MockIndexWriter : IndexWriter
+ {
+ public MockIndexWriter(Directory dir, IndexWriterConfig conf)
+ : base(dir, conf)
+ {
+ }
+
+ internal bool AfterWasCalled;
+ internal bool BeforeWasCalled;
+
+ protected override void DoAfterFlush()
+ {
+ AfterWasCalled = true;
+ }
+
+ protected override void DoBeforeFlush()
+ {
+ BeforeWasCalled = true;
+ }
+ }
+
+ // LUCENE-1222
+ [Test]
+ public virtual void TestDoBeforeAfterFlush()
+ {
+ Directory dir = NewDirectory();
+ MockIndexWriter w = new MockIndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ doc.Add(NewField("field", "a field", customType));
+ w.AddDocument(doc);
+ w.Commit();
+ Assert.IsTrue(w.BeforeWasCalled);
+ Assert.IsTrue(w.AfterWasCalled);
+ w.BeforeWasCalled = false;
+ w.AfterWasCalled = false;
+ w.DeleteDocuments(new Term("field", "field"));
+ w.Commit();
+ Assert.IsTrue(w.BeforeWasCalled);
+ Assert.IsTrue(w.AfterWasCalled);
+ w.Dispose();
+
+ IndexReader ir = DirectoryReader.Open(dir);
+ Assert.AreEqual(0, ir.NumDocs);
+ ir.Dispose();
+
+ dir.Dispose();
+ }
+
+ // LUCENE-1255
+ [Test]
+ public virtual void TestNegativePositions()
+ {
+ TokenStream tokens = new TokenStreamAnonymousInnerClassHelper(this);
+
+ Directory dir = NewDirectory();
+ IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ doc.Add(new TextField("field", tokens));
+ try
+ {
+ w.AddDocument(doc);
+ Assert.Fail("did not hit expected exception");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException iea)
+#pragma warning restore 168
+ {
+ // expected
+ }
+ w.Dispose();
+ dir.Dispose();
+ }
+
+ private class TokenStreamAnonymousInnerClassHelper : TokenStream
+ {
+ private readonly TestIndexWriter OuterInstance;
+
+ public TokenStreamAnonymousInnerClassHelper(TestIndexWriter outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ termAtt = AddAttribute<ICharTermAttribute>();
+ posIncrAtt = AddAttribute<IPositionIncrementAttribute>();
+ terms = Arrays.AsList("a", "b", "c").GetEnumerator();
+ first = true;
+ }
+
+ internal readonly ICharTermAttribute termAtt;
+ internal readonly IPositionIncrementAttribute posIncrAtt;
+
+ internal readonly IEnumerator<string> terms;
+ internal bool first;
+
+ public sealed override bool IncrementToken()
+ {
+ if (!terms.MoveNext())
+ {
+ return false;
+ }
+ ClearAttributes();
+ termAtt.Append(terms.Current);
+ posIncrAtt.PositionIncrement = first ? 0 : 1;
+ first = false;
+ return true;
+ }
+ }
+
+ // LUCENE-2529
+ [Test]
+ public virtual void TestPositionIncrementGapEmptyField()
+ {
+ Directory dir = NewDirectory();
+ MockAnalyzer analyzer = new MockAnalyzer(Random());
+ analyzer.PositionIncrementGap = 100;
+ IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.StoreTermVectors = true;
+ customType.StoreTermVectorPositions = true;
+ Field f = NewField("field", "", customType);
+ Field f2 = NewField("field", "crunch man", customType);
+ doc.Add(f);
+ doc.Add(f2);
+ w.AddDocument(doc);
+ w.Dispose();
+
+ IndexReader r = DirectoryReader.Open(dir);
+ Terms tpv = r.GetTermVectors(0).GetTerms("field");
+ TermsEnum termsEnum = tpv.GetIterator(null);
+ Assert.IsNotNull(termsEnum.Next());
+ DocsAndPositionsEnum dpEnum = termsEnum.DocsAndPositions(null, null);
+ Assert.IsNotNull(dpEnum);
+ Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ Assert.AreEqual(1, dpEnum.Freq);
+ Assert.AreEqual(100, dpEnum.NextPosition());
+
+ Assert.IsNotNull(termsEnum.Next());
+ dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
+ Assert.IsNotNull(dpEnum);
+ Assert.IsTrue(dpEnum.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ Assert.AreEqual(1, dpEnum.Freq);
+ Assert.AreEqual(101, dpEnum.NextPosition());
+ Assert.IsNull(termsEnum.Next());
+
+ r.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestDeadlock()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
+ Document doc = new Document();
+
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.StoreTermVectors = true;
+ customType.StoreTermVectorPositions = true;
+ customType.StoreTermVectorOffsets = true;
+
+ doc.Add(NewField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", customType));
+ writer.AddDocument(doc);
+ writer.AddDocument(doc);
+ writer.AddDocument(doc);
+ writer.Commit();
+ // index has 2 segments
+
+ Directory dir2 = NewDirectory();
+ IndexWriter writer2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ writer2.AddDocument(doc);
+ writer2.Dispose();
+
+ IndexReader r1 = DirectoryReader.Open(dir2);
+ writer.AddIndexes(r1, r1);
+ writer.Dispose();
+
+ IndexReader r3 = DirectoryReader.Open(dir);
+ Assert.AreEqual(5, r3.NumDocs);
+ r3.Dispose();
+
+ r1.Dispose();
+
+ dir2.Dispose();
+ dir.Dispose();
+ }
+
+ private class IndexerThreadInterrupt : ThreadClass
+ {
+ private readonly TestIndexWriter OuterInstance;
+
+ internal volatile bool Failed;
+ internal volatile bool Finish;
+
+ internal volatile bool AllowInterrupt = false;
+ internal readonly Random Random;
+ internal readonly Directory Adder;
+
+ internal IndexerThreadInterrupt(TestIndexWriter outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ this.Random = new Random(Random().Next());
+ // make a little directory for addIndexes
+ // LUCENE-2239: won't work with NIOFS/MMAP
+ Adder = new MockDirectoryWrapper(this.Random, new RAMDirectory());
+ IndexWriterConfig conf = OuterInstance.NewIndexWriterConfig(this.Random, TEST_VERSION_CURRENT, new MockAnalyzer(this.Random));
+ IndexWriter w = new IndexWriter(Adder, conf);
+ Document doc = new Document();
+ doc.Add(OuterInstance.NewStringField(this.Random, "id", "500", Field.Store.NO));
+ doc.Add(OuterInstance.NewField(this.Random, "field", "some prepackaged text contents", StoredTextType));
+ if (DefaultCodecSupportsDocValues())
+ {
+ doc.Add(new BinaryDocValuesField("binarydv", new BytesRef("500")));
+ doc.Add(new NumericDocValuesField("numericdv", 500));
+ doc.Add(new SortedDocValuesField("sorteddv", new BytesRef("500")));
+ }
+ if (DefaultCodecSupportsSortedSet())
+ {
+ doc.Add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("one")));
+ doc.Add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("two")));
+ }
+ w.AddDocument(doc);
+ doc = new Document();
+ doc.Add(OuterInstance.NewStringField(this.Random, "id", "501", Field.Store.NO));
+ doc.Add(OuterInstance.NewField(this.Random, "field", "some more contents", StoredTextType));
+ if (DefaultCodecSupportsDocValues())
+ {
+ doc.Add(new BinaryDocValuesField("binarydv", new BytesRef("501")));
+ doc.Add(new NumericDocValuesField("numericdv", 501));
+ doc.Add(new SortedDocValuesField("sorteddv", new BytesRef("501")));
+ }
+ if (DefaultCodecSupportsSortedSet())
+ {
+ doc.Add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("two")));
+ doc.Add(new SortedSetDocValuesField("sortedsetdv", new BytesRef("three")));
+ }
+ w.AddDocument(doc);
+ w.DeleteDocuments(new Term("id", "500"));
+ w.Dispose();
+ }
+
+ public override void Run()
+ {
+ // LUCENE-2239: won't work with NIOFS/MMAP
+ MockDirectoryWrapper dir = new MockDirectoryWrapper(Random, new RAMDirectory());
+
+ // When interrupt arrives in w.Dispose(), when it's
+ // writing liveDocs, this can lead to double-write of
+ // _X_N.del:
+ //dir.setPreventDoubleWrite(false);
+ IndexWriter w = null;
+ while (!Finish)
+ {
+ try
+ {
+ while (!Finish)
+ {
+ if (w != null)
+ {
+ // If interrupt arrives inside here, it's
+ // fine: we will cycle back and the first
+ // thing we do is try to close again,
+ // i.e. we'll never try to open a new writer
+ // until this one successfully closes:
+ w.Dispose();
+ w = null;
+ }
+ IndexWriterConfig conf = OuterInstance.NewIndexWriterConfig(Random, TEST_VERSION_CURRENT, new MockAnalyzer(Random)).SetMaxBufferedDocs(2);
+ w = new IndexWriter(dir, conf);
+
+ Document doc = new Document();
+ Field idField = OuterInstance.NewStringField(Random, "id", "", Field.Store.NO);
+ Field binaryDVField = null;
+ Field numericDVField = null;
+ Field sortedDVField = null;
+ Field sortedSetDVField = new SortedSetDocValuesField("sortedsetdv", new BytesRef());
+ doc.Add(idField);
+ doc.Add(OuterInstance.NewField(Random, "field", "some text contents", StoredTextType));
+ if (DefaultCodecSupportsDocValues())
+ {
+ binaryDVField = new BinaryDocValuesField("binarydv", new BytesRef());
+ numericDVField = new NumericDocValuesField("numericdv", 0);
+ sortedDVField = new SortedDocValuesField("sorteddv", new BytesRef());
+ doc.Add(binaryDVField);
+ doc.Add(numericDVField);
+ doc.Add(sortedDVField);
+ }
+ if (DefaultCodecSupportsSortedSet())
+ {
+ doc.Add(sortedSetDVField);
+ }
+ for (int i = 0; i < 100; i++)
+ {
+ idField.SetStringValue(Convert.ToString(i));
+ if (DefaultCodecSupportsDocValues())
+ {
+ binaryDVField.SetBytesValue(new BytesRef(idField.GetStringValue()));
+ numericDVField.SetInt64Value(i);
+ sortedDVField.SetBytesValue(new BytesRef(idField.GetStringValue()));
+ }
+ sortedSetDVField.SetBytesValue(new BytesRef(idField.GetStringValue()));
+ int action = Random.Next(100);
+ if (action == 17)
+ {
+ w.AddIndexes(Adder);
+ }
+ else if (action % 30 == 0)
+ {
+ w.DeleteAll();
+ }
+ else if (action % 2 == 0)
+ {
+ w.UpdateDocument(new Term("id", idField.GetStringValue()), doc);
+ }
+ else
+ {
+ w.AddDocument(doc);
+ }
+ if (Random.Next(3) == 0)
+ {
+ IndexReader r = null;
+ try
+ {
+ r = DirectoryReader.Open(w, Random.NextBoolean());
+ if (Random.NextBoolean() && r.MaxDoc > 0)
+ {
+ int docid = Random.Next(r.MaxDoc);
+ w.TryDeleteDocument(r, docid);
+ }
+ }
+ finally
+ {
+ IOUtils.CloseWhileHandlingException(r);
+ }
+ }
+ if (i % 10 == 0)
+ {
+ w.Commit();
+ }
+ if (Random.Next(50) == 0)
+ {
+ w.ForceMerge(1);
+ }
+ }
+ w.Dispose();
+ w = null;
+ DirectoryReader.Open(dir).Dispose();
+
+ // Strangely, if we interrupt a thread before
+ // all classes are loaded, the class loader
+ // seems to do scary things with the interrupt
+ // status. In java 1.5, it'll throw an
+ // incorrect ClassNotFoundException. In java
+ // 1.6, it'll silently clear the interrupt.
+ // So, on first iteration through here we
+ // don't open ourselves up for interrupts
+ // until we've done the above loop.
+ AllowInterrupt = true;
+ }
+ }
+#if !NETSTANDARD
+ catch (ThreadInterruptedException re)
+ {
+ // NOTE: important to leave this verbosity/noise
+ // on!! this test doesn't repro easily so when
+ // Jenkins hits a fail we need to study where the
+ // interrupts struck!
+ Console.WriteLine("TEST: got interrupt");
+ Console.WriteLine(re.StackTrace);
+ Exception e = re.InnerException;
+ Assert.IsTrue(e is ThreadInterruptedException);
+ if (Finish)
+ {
+ break;
+ }
+ }
+#endif
+ catch (Exception t)
+ {
+ Console.WriteLine("FAILED; unexpected exception");
+ Console.WriteLine(t.StackTrace);
+ Failed = true;
+ break;
+ }
+ }
+
+ if (!Failed)
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine("TEST: now rollback");
+ }
+ // clear interrupt state:
+ //Thread.interrupted();
+ if (w != null)
+ {
+ try
+ {
+ w.Rollback();
+ }
+ catch (IOException ioe)
+ {
+ throw new Exception(ioe.Message, ioe);
+ }
+ }
+
+ try
+ {
+ TestUtil.CheckIndex(dir);
+ }
+ catch (Exception e)
+ {
+ Failed = true;
+ Console.WriteLine("CheckIndex FAILED: unexpected exception");
+ Console.WriteLine(e.StackTrace);
+ }
+ try
+ {
+ IndexReader r = DirectoryReader.Open(dir);
+ //System.out.println("doc count=" + r.NumDocs);
+ r.Dispose();
+ }
+ catch (Exception e)
+ {
+ Failed = true;
+ Console.WriteLine("DirectoryReader.open FAILED: unexpected exception");
+ Console.WriteLine(e.StackTrace);
+ }
+ }
+ try
+ {
+ IOUtils.Close(dir);
+ }
+ catch (IOException e)
+ {
+ throw new Exception(e.Message, e);
+ }
+ try
+ {
+ IOUtils.Close(Adder);
+ }
+ catch (IOException e)
+ {
+ throw new Exception(e.Message, e);
+ }
+ }
+ }
+
+ [Test]
+ public virtual void TestThreadInterruptDeadlock()
+ {
+ IndexerThreadInterrupt t = new IndexerThreadInterrupt(this);
+ t.SetDaemon(true);
+ t.Start();
+
+ // Force class loader to load ThreadInterruptedException
+ // up front... else we can see a false failure if 2nd
+ // interrupt arrives while class loader is trying to
+ // init this class (in servicing a first interrupt):
+ //Assert.IsTrue((new ThreadInterruptedException(new Exception("Thread interrupted"))).InnerException is ThreadInterruptedException);
+
+ // issue 300 interrupts to child thread
+ int numInterrupts = AtLeast(300);
+ int i = 0;
+ while (i < numInterrupts)
+ {
+ // TODO: would be nice to also sometimes interrupt the
+ // CMS merge threads too ...
+ Thread.Sleep(10);
+ if (t.AllowInterrupt)
+ {
+ i++;
+ t.Interrupt();
+ }
+ if (!t.IsAlive)
+ {
+ break;
+ }
+ }
+ t.Finish = true;
+ t.Join();
+ Assert.IsFalse(t.Failed);
+ }
+
+ /// <summary>
+ /// testThreadInterruptDeadlock but with 2 indexer threads </summary>
+ [Test]
+ public virtual void TestTwoThreadsInterruptDeadlock()
+ {
+ IndexerThreadInterrupt t1 = new IndexerThreadInterrupt(this);
+ t1.SetDaemon(true);
+ t1.Start();
+
+ IndexerThreadInterrupt t2 = new IndexerThreadInterrupt(this);
+ t2.SetDaemon(true);
+ t2.Start();
+
+ // Force class loader to load ThreadInterruptedException
+ // up front... else we can see a false failure if 2nd
+ // interrupt arrives while class loader is trying to
+ // init this class (in servicing a first interrupt):
+ // C# does not have the late load problem.
+ //Assert.IsTrue((new ThreadInterruptedException(new Exception("Thread interrupted"))).InnerException is ThreadInterruptedException);
+
+ // issue 300 interrupts to child thread
+ int numInterrupts = AtLeast(300);
+ int i = 0;
+ while (i < numInterrupts)
+ {
+ // TODO: would be nice to also sometimes interrupt the
+ // CMS merge threads too ...
+ Thread.Sleep(10);
+ IndexerThreadInterrupt t = Random().NextBoolean() ? t1 : t2;
+ if (t.AllowInterrupt)
+ {
+ i++;
+ t.Interrupt();
+ }
+ if (!t1.IsAlive && !t2.IsAlive)
+ {
+ break;
+ }
+ }
+ t1.Finish = true;
+ t2.Finish = true;
+ t1.Join();
+ t2.Join();
+ Assert.IsFalse(t1.Failed);
+ Assert.IsFalse(t2.Failed);
+ }
+
+ [Test]
+ public virtual void TestIndexStoreCombos()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ var b = new byte[50];
+ for (int i = 0; i < 50; i++)
+ {
+ b[i] = (byte)(i + 77);
+ }
+
+ Document doc = new Document();
+
+ FieldType customType = new FieldType(StoredField.TYPE);
+ customType.IsTokenized = true;
+
+ Field f = new Field("binary", b, 10, 17, customType);
+ customType.IsIndexed = true;
+ f.SetTokenStream(new MockTokenizer(new StringReader("doc1field1"), MockTokenizer.WHITESPACE, false));
+
+ FieldType customType2 = new FieldType(TextField.TYPE_STORED);
+
+ Field f2 = NewField("string", "value", customType2);
+ f2.SetTokenStream(new MockTokenizer(new StringReader("doc1field2"), MockTokenizer.WHITESPACE, false));
+ doc.Add(f);
+ doc.Add(f2);
+ w.AddDocument(doc);
+
+ // add 2 docs to test in-memory merging
+ f.SetTokenStream(new MockTokenizer(new StringReader("doc2field1"), MockTokenizer.WHITESPACE, false));
+ f2.SetTokenStream(new MockTokenizer(new StringReader("doc2field2"), MockTokenizer.WHITESPACE, false));
+ w.AddDocument(doc);
+
+ // force segment flush so we can force a segment merge with doc3 later.
+ w.Commit();
+
+ f.SetTokenStream(new MockTokenizer(new StringReader("doc3field1"), MockTokenizer.WHITESPACE, false));
+ f2.SetTokenStream(new MockTokenizer(new StringReader("doc3field2"), MockTokenizer.WHITESPACE, false));
+
+ w.AddDocument(doc);
+ w.Commit();
+ w.ForceMerge(1); // force segment merge.
+ w.Dispose();
+
+ IndexReader ir = DirectoryReader.Open(dir);
+ Document doc2 = ir.Document(0);
+ IIndexableField f3 = doc2.GetField("binary");
+ b = f3.GetBinaryValue().Bytes;
+ Assert.IsTrue(b != null);
+ Assert.AreEqual(17, b.Length, 17);
+ Assert.AreEqual(87, b[0]);
+
+ Assert.IsTrue(ir.Document(0).GetField("binary").GetBinaryValue() != null);
+ Assert.IsTrue(ir.Document(1).GetField("binary").GetBinaryValue() != null);
+ Assert.IsTrue(ir.Document(2).GetField("binary").GetBinaryValue() != null);
+
+ Assert.AreEqual("value", ir.Document(0).Get("string"));
+ Assert.AreEqual("value", ir.Document(1).Get("string"));
+ Assert.AreEqual("value", ir.Document(2).Get("string"));
+
+ // test that the terms were indexed.
+ Assert.IsTrue(TestUtil.Docs(Random(), ir, "binary", new BytesRef("doc1field1"), null, null, DocsEnum.FLAG_NONE).NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ Assert.IsTrue(TestUtil.Docs(Random(), ir, "binary", new BytesRef("doc2field1"), null, null, DocsEnum.FLAG_NONE).NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ Assert.IsTrue(TestUtil.Docs(Random(), ir, "binary", new BytesRef("doc3field1"), null, null, DocsEnum.FLAG_NONE).NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ Assert.IsTrue(TestUtil.Docs(Random(), ir, "string", new BytesRef("doc1field2"), null, null, DocsEnum.FLAG_NONE).NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ Assert.IsTrue(TestUtil.Docs(Random(), ir, "string", new BytesRef("doc2field2"), null, null, DocsEnum.FLAG_NONE).NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ Assert.IsTrue(TestUtil.Docs(Random(), ir, "string", new BytesRef("doc3field2"), null, null, DocsEnum.FLAG_NONE).NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+
+ ir.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestNoDocsIndex()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ writer.AddDocument(new Document());
+ writer.Dispose();
+
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestIndexDivisor()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig config = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ config.SetTermIndexInterval(2);
+ IndexWriter w = new IndexWriter(dir, config);
+ StringBuilder s = new StringBuilder();
+ // must be > 256
+ for (int i = 0; i < 300; i++)
+ {
+ s.Append(' ').Append(i);
+ }
+ Document d = new Document();
+ Field f = NewTextField("field", s.ToString(), Field.Store.NO);
+ d.Add(f);
+ w.AddDocument(d);
+
+ AtomicReader r = GetOnlySegmentReader(w.Reader);
+ TermsEnum t = r.Fields.GetTerms("field").GetIterator(null);
+ int count = 0;
+ while (t.Next() != null)
+ {
+ DocsEnum docs = TestUtil.Docs(Random(), t, null, null, DocsEnum.FLAG_NONE);
+ Assert.AreEqual(0, docs.NextDoc());
+ Assert.AreEqual(DocIdSetIterator.NO_MORE_DOCS, docs.NextDoc());
+ count++;
+ }
+ Assert.AreEqual(300, count);
+ r.Dispose();
+ w.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestDeleteUnusedFiles()
+ {
+ for (int iter = 0; iter < 2; iter++)
+ {
+ Directory dir = NewMockDirectory(); // relies on windows semantics
+
+ MergePolicy mergePolicy = NewLogMergePolicy(true);
+
+ // this test expects all of its segments to be in CFS
+ mergePolicy.NoCFSRatio = 1.0;
+ mergePolicy.MaxCFSSegmentSizeMB = double.PositiveInfinity;
+
+ IndexWriter w = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMergePolicy(mergePolicy).SetUseCompoundFile(true));
+ Document doc = new Document();
+ doc.Add(NewTextField("field", "go", Field.Store.NO));
+ w.AddDocument(doc);
+ DirectoryReader r;
+ if (iter == 0)
+ {
+ // use NRT
+ r = w.Reader;
+ }
+ else
+ {
+ // don't use NRT
+ w.Commit();
+ r = DirectoryReader.Open(dir);
+ }
+
+ IList<string> files = new List<string>(Arrays.AsList(dir.ListAll()));
+
+ // RAMDir won't have a write.lock, but fs dirs will:
+ files.Remove("write.lock");
+
+ Assert.IsTrue(files.Contains("_0.cfs"));
+ Assert.IsTrue(files.Contains("_0.cfe"));
+ Assert.IsTrue(files.Contains("_0.si"));
+ if (iter == 1)
+ {
+ // we run a full commit so there should be a segments file etc.
+ Assert.IsTrue(files.Contains("segments_1"));
+ Assert.IsTrue(files.Contains("segments.gen"));
+ Assert.AreEqual(files.Count, 5, files.ToString());
+ }
+ else
+ {
+ // this is an NRT reopen - no segments files yet
+
+ Assert.AreEqual(files.Count, 3, files.ToString());
+ }
+ w.AddDocument(doc);
+ w.ForceMerge(1);
+ if (iter == 1)
+ {
+ w.Commit();
+ }
+ IndexReader r2 = DirectoryReader.OpenIfChanged(r);
+ Assert.IsNotNull(r2);
+ Assert.IsTrue(r != r2);
+ files = Arrays.AsList(dir.ListAll());
+
+ // NOTE: here we rely on "Windows" behavior, ie, even
+ // though IW wanted to delete _0.cfs since it was
+ // merged away, because we have a reader open
+ // against this file, it should still be here:
+ Assert.IsTrue(files.Contains("_0.cfs"));
+ // forceMerge created this
+ //Assert.IsTrue(files.Contains("_2.cfs"));
+ w.DeleteUnusedFiles();
+
+ files = Arrays.AsList(dir.ListAll());
+ // r still holds this file open
+ Assert.IsTrue(files.Contains("_0.cfs"));
+ //Assert.IsTrue(files.Contains("_2.cfs"));
+
+ r.Dispose();
+ if (iter == 0)
+ {
+ // on closing NRT reader, it calls writer.deleteUnusedFiles
+ files = Arrays.AsList(dir.ListAll());
+ Assert.IsFalse(files.Contains("_0.cfs"));
+ }
+ else
+ {
+ // now writer can remove it
+ w.DeleteUnusedFiles();
+ files = Arrays.AsList(dir.ListAll());
+ Assert.IsFalse(files.Contains("_0.cfs"));
+ }
+ //Assert.IsTrue(files.Contains("_2.cfs"));
+
+ w.Dispose();
+ r2.Dispose();
+
+ dir.Dispose();
+ }
+ }
+
+ [Test]
+ public virtual void TestDeleteUnsedFiles2()
+ {
+ // Validates that iw.DeleteUnusedFiles() also deletes unused index commits
+ // in case a deletion policy which holds onto commits is used.
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetIndexDeletionPolicy(new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy())));
+ SnapshotDeletionPolicy sdp = (SnapshotDeletionPolicy)writer.Config.IndexDeletionPolicy;
+
+ // First commit
+ Document doc = new Document();
+
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.StoreTermVectors = true;
+ customType.StoreTermVectorPositions = true;
+ customType.StoreTermVectorOffsets = true;
+
+ doc.Add(NewField("c", "val", customType));
+ writer.AddDocument(doc);
+ writer.Commit();
+ Assert.AreEqual(1, DirectoryReader.ListCommits(dir).Count);
+
+ // Keep that commit
+ IndexCommit id = sdp.Snapshot();
+
+ // Second commit - now KeepOnlyLastCommit cannot delete the prev commit.
+ doc = new Document();
+ doc.Add(NewField("c", "val", customType));
+ writer.AddDocument(doc);
+ writer.Commit();
+ Assert.AreEqual(2, DirectoryReader.ListCommits(dir).Count);
+
+ // Should delete the unreferenced commit
+ sdp.Release(id);
+ writer.DeleteUnusedFiles();
+ Assert.AreEqual(1, DirectoryReader.ListCommits(dir).Count);
+
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestEmptyFSDirWithNoLock()
+ {
+ // Tests that if FSDir is opened w/ a NoLockFactory (or SingleInstanceLF),
+ // then IndexWriter ctor succeeds. Previously (LUCENE-2386) it failed
+ // when listAll() was called in IndexFileDeleter.
+ Directory dir = NewFSDirectory(CreateTempDir("emptyFSDirNoLock"), NoLockFactory.GetNoLockFactory());
+ (new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())))).Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestEmptyDirRollback()
+ {
+ // TODO: generalize this test
+ //AssumeFalse("test makes assumptions about file counts", Codec.Default is SimpleTextCodec);
+
+ // Tests that if IW is created over an empty Directory, some documents are
+ // indexed, flushed (but not committed) and then IW rolls back, then no
+ // files are left in the Directory.
+ Directory dir = NewDirectory();
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetMergePolicy(NewLogMergePolicy()).SetUseCompoundFile(false));
+ string[] files = dir.ListAll();
+
+ // Creating over empty dir should not create any files,
+ // or, at most the write.lock file
+ int extraFileCount;
+ if (files.Length == 1)
+ {
+ Assert.IsTrue(files[0].EndsWith("write.lock"));
+ extraFileCount = 1;
+ }
+ else
+ {
+ Assert.AreEqual(0, files.Length);
+ extraFileCount = 0;
+ }
+
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.StoreTermVectors = true;
+ customType.StoreTermVectorPositions = true;
+ customType.StoreTermVectorOffsets = true;
+ // create as many files as possible
+ doc.Add(NewField("c", "val", customType));
+ writer.AddDocument(doc);
+ // Adding just one document does not call flush yet.
+ int computedExtraFileCount = 0;
+ foreach (string file in dir.ListAll())
+ {
+ if (file.LastIndexOf('.') < 0 || !Arrays.AsList("fdx", "fdt", "tvx", "tvd", "tvf").Contains(file.Substring(file.LastIndexOf('.') + 1)))
+ // don't count stored fields and term vectors in
+ {
+ ++computedExtraFileCount;
+ }
+ }
+ Assert.AreEqual(extraFileCount, computedExtraFileCount, "only the stored and term vector files should exist in the directory");
+
+ doc = new Document();
+ doc.Add(NewField("c", "val", customType));
+ writer.AddDocument(doc);
+
+ // The second document should cause a flush.
+ Assert.IsTrue(dir.ListAll().Length > 5 + extraFileCount, "flush should have occurred and files should have been created");
+
+ // After rollback, IW should remove all files
+ writer.Rollback();
+ string[] allFiles = dir.ListAll();
+ Assert.IsTrue(allFiles.Length == 0 || Arrays.Equals(allFiles, new string[] { IndexWriter.WRITE_LOCK_NAME }), "no files should exist in the directory after rollback");
+
+ // Since we rolled-back above, that close should be a no-op
+ writer.Dispose();
+ allFiles = dir.ListAll();
+ Assert.IsTrue(allFiles.Length == 0 || Arrays.Equals(allFiles, new string[] { IndexWriter.WRITE_LOCK_NAME }), "expected a no-op close after IW.Rollback()");
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestNoSegmentFile()
+ {
+ BaseDirectoryWrapper dir = NewDirectory();
+ dir.SetLockFactory(NoLockFactory.GetNoLockFactory());
+ IndexWriter w = new IndexWriter(dir, (IndexWriterConfig)NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2));
+
+ Document doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.StoreTermVectors = true;
+ customType.StoreTermVectorPositions = true;
+ customType.StoreTermVectorOffsets = true;
+ doc.Add(NewField("c", "val", customType));
+ w.AddDocument(doc);
+ w.AddDocument(doc);
+ IndexWriter w2 = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetMaxBufferedDocs(2).SetOpenMode(OpenMode.CREATE));
+
+ w2.Dispose();
+ // If we don't do that, the test fails on Windows
+ w.Rollback();
+
+ // this test leaves only segments.gen, which causes
+ // DirectoryReader.indexExists to return true:
+ dir.CheckIndexOnClose = false;
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestNoUnwantedTVFiles()
+ {
+ Directory dir = NewDirectory();
+ IndexWriter indexWriter = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetRAMBufferSizeMB(0.01).SetMergePolicy(NewLogMergePolicy()));
+ indexWriter.Config.MergePolicy.NoCFSRatio = 0.0;
+
+ string BIG = "alskjhlaksjghlaksjfhalksvjepgjioefgjnsdfjgefgjhelkgjhqewlrkhgwlekgrhwelkgjhwelkgrhwlkejg";
+ BIG = BIG + BIG + BIG + BIG;
+
+ FieldType customType = new FieldType(TextField.TYPE_STORED);
+ customType.OmitNorms = true;
+ FieldType customType2 = new FieldType(TextField.TYPE_STORED);
+ customType2.IsTokenized = false;
+ FieldType customType3 = new FieldType(TextField.TYPE_STORED);
+ customType3.IsTokenized = false;
+ customType3.OmitNorms = true;
+
+ for (int i = 0; i < 2; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new Field("id", Convert.ToString(i) + BIG, customType3));
+ doc.Add(new Field("str", Convert.ToString(i) + BIG, customType2));
+ doc.Add(new Field("str2", Convert.ToString(i) + BIG, StoredTextType));
+ doc.Add(new Field("str3", Convert.ToString(i) + BIG, customType));
+ indexWriter.AddDocument(doc);
+ }
+
+ indexWriter.Dispose();
+
+ TestUtil.CheckIndex(dir);
+
+ AssertNoUnreferencedFiles(dir, "no tv files");
+ DirectoryReader r0 = DirectoryReader.Open(dir);
+ foreach (AtomicReaderContext ctx in r0.Leaves)
+ {
+ SegmentReader sr = (SegmentReader)ctx.Reader;
+ Assert.IsFalse(sr.FieldInfos.HasVectors);
+ }
+
+ r0.Dispose();
+ dir.Dispose();
+ }
+
+ internal sealed class StringSplitAnalyzer : Analyzer
+ {
+ protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+ {
+ return new TokenStreamComponents(new StringSplitTokenizer(reader));
+ }
+ }
+
+ private class StringSplitTokenizer : Tokenizer
+ {
+ private string[] Tokens;
+ private int Upto;
+ private readonly ICharTermAttribute TermAtt;
+
+ public StringSplitTokenizer(TextReader r)
+ : base(r)
+ {
+ TermAtt = AddAttribute<ICharTermAttribute>();
+ try
+ {
+ SetReader(r);
+ }
+ catch (IOException e)
+ {
+ throw new Exception(e.Message, e);
+ }
+ }
+
+ public sealed override bool IncrementToken()
+ {
+ ClearAttributes();
+ if (Upto < Tokens.Length)
+ {
+ TermAtt.SetEmpty();
+ TermAtt.Append(Tokens[Upto]);
+ Upto++;
+ return true;
+ }
+ else
+ {
+ return false;
+ }
+ }
+
+ public override void Reset()
+ {
+ base.Reset();
+ this.Upto = 0;
+ StringBuilder b = new StringBuilder();
+ char[] buffer = new char[1024];
+ int n;
+ while ((n = m_input.Read(buffer, 0, buffer.Length)) > 0)
+ {
+ b.Append(buffer, 0, n);
+ }
+ this.Tokens = b.ToString().Split(' ');
+ }
+ }
+
+ /// <summary>
+ /// Make sure we skip wicked long terms.
+ /// </summary>
+ [Test]
+ public virtual void TestWickedLongTerm()
+ {
+ Directory dir = NewDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(Random(), dir, new StringSplitAnalyzer(), Similarity, TimeZone);
+
+ char[] chars = new char[DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8];
+ Arrays.Fill(chars, 'x');
+ Document doc = new Document();
+ string bigTerm = new string(chars);
+ BytesRef bigTermBytesRef = new BytesRef(bigTerm);
+
+ // this contents produces a too-long term:
+ string contents = "abc xyz x" + bigTerm + " another term";
+ doc.Add(new TextField("content", contents, Field.Store.NO));
+ try
+ {
+ w.AddDocument(doc);
+ Assert.Fail("should have hit exception");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException iae)
+#pragma warning restore 168
+ {
+ // expected
+ }
+
+ // Make sure we can add another normal document
+ doc = new Document();
+ doc.Add(new TextField("content", "abc bbb ccc", Field.Store.NO));
+ w.AddDocument(doc);
+
+ // So we remove the deleted doc:
+ w.ForceMerge(1);
+
+ IndexReader reader = w.Reader;
+ w.Dispose();
+
+ // Make sure all terms < max size were indexed
+ Assert.AreEqual(1, reader.DocFreq(new Term("content", "abc")));
+ Assert.AreEqual(1, reader.DocFreq(new Term("content", "bbb")));
+ Assert.AreEqual(0, reader.DocFreq(new Term("content", "term")));
+
+ // Make sure the doc that has the massive term is NOT in
+ // the index:
+ Assert.AreEqual(1, reader.NumDocs, "document with wicked long term is in the index!");
+
+ reader.Dispose();
+ dir.Dispose();
+ dir = NewDirectory();
+
+ // Make sure we can add a document with exactly the
+ // maximum length term, and search on that term:
+ doc = new Document();
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.IsTokenized = false;
+ Field contentField = new Field("content", "", customType);
+ doc.Add(contentField);
+
+ w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
+
+ contentField.SetStringValue("other");
+ w.AddDocument(doc);
+
+ contentField.SetStringValue("term");
+ w.AddDocument(doc);
+
+ contentField.SetStringValue(bigTerm);
+ w.AddDocument(doc);
+
+ contentField.SetStringValue("zzz");
+ w.AddDocument(doc);
+
+ reader = w.Reader;
+ w.Dispose();
+ Assert.AreEqual(1, reader.DocFreq(new Term("content", bigTerm)));
+
+ SortedDocValues dti = FieldCache.DEFAULT.GetTermsIndex(SlowCompositeReaderWrapper.Wrap(reader), "content", (float)Random().NextDouble() * PackedInt32s.FAST);
+ Assert.AreEqual(4, dti.ValueCount);
+ BytesRef br = new BytesRef();
+ dti.LookupOrd(2, br);
+ Assert.AreEqual(bigTermBytesRef, br);
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ // LUCENE-3183
+ [Test]
+ public virtual void TestEmptyFieldNameTIIOne()
+ {
+ Directory dir = NewDirectory();
+ IndexWriterConfig iwc = NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ iwc.SetTermIndexInterval(1);
+ iwc.SetReaderTermsIndexDivisor(1);
+ IndexWriter writer = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ doc.Add(NewTextField("", "a b c", Field.Store.NO));
+ writer.AddDocument(doc);
+ writer.Dispose();
+ dir.Dispose();
+ }
+
+ [Test]
+ public virtual void TestDeleteAllNRTLeftoverFiles()
+ {
+ Directory d = new MockDirectoryWrapper(Random(), new RAMDirectory());
+ IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ for (int i = 0; i < 20; i++)
+ {
+ for (int j = 0; j < 100; ++j)
+ {
+ w.AddDocument(doc);
+ }
+ w.Commit();
+ DirectoryReader.Open(w, true).Dispose();
+
+ w.DeleteAll();
+ w.Commit();
+ // Make sure we accumulate no files except for empty
+ // segments_N and segments.gen:
+ Assert.IsTrue(d.ListAll().Length <= 2);
+ }
+
+ w.Dispose();
+ d.Dispose();
+ }
+
+ [Test]
+ public virtual void TestNRTReaderVersion()
+ {
+ Directory d = new MockDirectoryWrapper(Random(), new RAMDirectory());
+ IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ doc.Add(NewStringField("id", "0", Field.Store.YES));
+ w.AddDocument(doc);
+ DirectoryReader r = w.Reader;
+ long version = r.Version;
+ r.Dispose();
+
+ w.AddDocument(doc);
+ r = w.Reader;
+ long version2 = r.Version;
+ r.Dispose();
+ Debug.Assert(version2 > version);
+
+ w.DeleteDocuments(new Term("id", "0"));
+ r = w.Reader;
+ w.Dispose();
+ long version3 = r.Version;
+ r.Dispose();
+ Debug.Assert(version3 > version2);
+ d.Dispose();
+ }
+
+ [Test]
+ public virtual void TestWhetherDeleteAllDeletesWriteLock()
+ {
+ Directory d = NewFSDirectory(CreateTempDir("TestIndexWriter.testWhetherDeleteAllDeletesWriteLock"
<TRUNCATED>