You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/02/26 23:37:03 UTC
[15/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests:
Removed \core directory and put its contents in root directory
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestTransactions.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestTransactions.cs b/src/Lucene.Net.Tests/Index/TestTransactions.cs
new file mode 100644
index 0000000..627b21c
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestTransactions.cs
@@ -0,0 +1,336 @@
+using System;
+using System.Threading;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Support;
+ using NUnit.Framework;
+ using System.IO;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using English = Lucene.Net.Util.English;
+ using Field = Field;
+ using FieldType = FieldType;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using MockDirectoryWrapper = Lucene.Net.Store.MockDirectoryWrapper;
+ using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+ using StringField = StringField;
+
+ [TestFixture]
+ public class TestTransactions : LuceneTestCase
+ {
+ private static volatile bool DoFail;
+
+ private class RandomFailure : MockDirectoryWrapper.Failure
+ {
+ private readonly TestTransactions OuterInstance;
+
+ public RandomFailure(TestTransactions outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ }
+
+ public override void Eval(MockDirectoryWrapper dir)
+ {
+ if (TestTransactions.DoFail && Random().Next() % 10 <= 3)
+ {
+ throw new IOException("now failing randomly but on purpose");
+ }
+ }
+ }
+
+ private abstract class TimedThread : ThreadClass
+ {
+ internal volatile bool Failed;
+ internal static float RUN_TIME_MSEC = AtLeast(500);
+ internal TimedThread[] AllThreads;
+
+ public abstract void DoWork();
+
+ internal TimedThread(TimedThread[] threads)
+ {
+ this.AllThreads = threads;
+ }
+
+ public override void Run()
+ {
+ long stopTime = Environment.TickCount + (long)(RUN_TIME_MSEC);
+
+ try
+ {
+ do
+ {
+ if (AnyErrors())
+ {
+ break;
+ }
+ DoWork();
+ } while (Environment.TickCount < stopTime);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine(Thread.CurrentThread + ": exc");
+ Console.Error.WriteLine(e.StackTrace);
+ Failed = true;
+ }
+ }
+
+ internal virtual bool AnyErrors()
+ {
+ for (int i = 0; i < AllThreads.Length; i++)
+ {
+ if (AllThreads[i] != null && AllThreads[i].Failed)
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+
+ private class IndexerThread : TimedThread
+ {
+ private readonly TestTransactions OuterInstance;
+ private IConcurrentMergeScheduler _scheduler1;
+ private IConcurrentMergeScheduler _scheduler2;
+ internal Directory Dir1;
+ internal Directory Dir2;
+ internal object @lock;
+ internal int NextID;
+
+ public IndexerThread(TestTransactions outerInstance, object @lock,
+ Directory dir1, Directory dir2,
+ IConcurrentMergeScheduler scheduler1, IConcurrentMergeScheduler scheduler2,
+ TimedThread[] threads)
+ : base(threads)
+ {
+ _scheduler1 = scheduler1;
+ _scheduler2 = scheduler2;
+ this.OuterInstance = outerInstance;
+ this.@lock = @lock;
+ this.Dir1 = dir1;
+ this.Dir2 = dir2;
+ }
+
+ public override void DoWork()
+ {
+ var config = OuterInstance.NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
+ .SetMaxBufferedDocs(3)
+ .SetMergeScheduler(_scheduler1)
+ .SetMergePolicy(NewLogMergePolicy(2));
+ IndexWriter writer1 = new IndexWriter(Dir1, config);
+ ((IConcurrentMergeScheduler)writer1.Config.MergeScheduler).SetSuppressExceptions();
+
+ // Intentionally use different params so flush/merge
+ // happen @ different times
+ var config2 = OuterInstance.NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))
+ .SetMaxBufferedDocs(2)
+ .SetMergeScheduler(_scheduler2)
+ .SetMergePolicy(NewLogMergePolicy(3));
+ IndexWriter writer2 = new IndexWriter(Dir2, config2);
+ ((IConcurrentMergeScheduler)writer2.Config.MergeScheduler).SetSuppressExceptions();
+
+ Update(writer1);
+ Update(writer2);
+
+ DoFail = true;
+ try
+ {
+ lock (@lock)
+ {
+ try
+ {
+ writer1.PrepareCommit();
+ }
+ catch (Exception)
+ {
+ writer1.Rollback();
+ writer2.Rollback();
+ return;
+ }
+ try
+ {
+ writer2.PrepareCommit();
+ }
+ catch (Exception)
+ {
+ writer1.Rollback();
+ writer2.Rollback();
+ return;
+ }
+
+ writer1.Commit();
+ writer2.Commit();
+ }
+ }
+ finally
+ {
+ DoFail = false;
+ }
+
+ writer1.Dispose();
+ writer2.Dispose();
+ }
+
+ public virtual void Update(IndexWriter writer)
+ {
+ // Add 10 docs:
+ FieldType customType = new FieldType(StringField.TYPE_NOT_STORED);
+ customType.StoreTermVectors = true;
+ for (int j = 0; j < 10; j++)
+ {
+ Document d = new Document();
+ int n = Random().Next();
+ d.Add(OuterInstance.NewField("id", Convert.ToString(NextID++), customType));
+ d.Add(OuterInstance.NewTextField("contents", English.IntToEnglish(n), Field.Store.NO));
+ writer.AddDocument(d);
+ }
+
+ // Delete 5 docs:
+ int deleteID = NextID - 1;
+ for (int j = 0; j < 5; j++)
+ {
+ writer.DeleteDocuments(new Term("id", "" + deleteID));
+ deleteID -= 2;
+ }
+ }
+ }
+
+ private class SearcherThread : TimedThread
+ {
+ internal Directory Dir1;
+ internal Directory Dir2;
+ internal object @lock;
+
+ public SearcherThread(object @lock, Directory dir1, Directory dir2, TimedThread[] threads)
+ : base(threads)
+ {
+ this.@lock = @lock;
+ this.Dir1 = dir1;
+ this.Dir2 = dir2;
+ }
+
+ public override void DoWork()
+ {
+ IndexReader r1 = null, r2 = null;
+ lock (@lock)
+ {
+ try
+ {
+ r1 = DirectoryReader.Open(Dir1);
+ r2 = DirectoryReader.Open(Dir2);
+ }
+ catch (IOException e)
+ {
+ if (!e.Message.Contains("on purpose"))
+ {
+ throw e;
+ }
+ if (r1 != null)
+ {
+ r1.Dispose();
+ }
+ if (r2 != null)
+ {
+ r2.Dispose();
+ }
+ return;
+ }
+ }
+ if (r1.NumDocs != r2.NumDocs)
+ {
+ throw new Exception("doc counts differ: r1=" + r1.NumDocs + " r2=" + r2.NumDocs);
+ }
+ r1.Dispose();
+ r2.Dispose();
+ }
+ }
+
+ public virtual void InitIndex(Directory dir)
+ {
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ for (int j = 0; j < 7; j++)
+ {
+ Document d = new Document();
+ int n = Random().Next();
+ d.Add(NewTextField("contents", English.IntToEnglish(n), Field.Store.NO));
+ writer.AddDocument(d);
+ }
+ writer.Dispose();
+ }
+
+ [Test, Sequential]
+ public virtual void TestTransactions_Mem(
+ [ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler1,
+ [ValueSource(typeof(ConcurrentMergeSchedulers), "Values")]IConcurrentMergeScheduler scheduler2)
+ {
+ Console.WriteLine("Start test");
+ // we cant use non-ramdir on windows, because this test needs to double-write.
+ MockDirectoryWrapper dir1 = new MockDirectoryWrapper(Random(), new RAMDirectory());
+ MockDirectoryWrapper dir2 = new MockDirectoryWrapper(Random(), new RAMDirectory());
+ dir1.PreventDoubleWrite = false;
+ dir2.PreventDoubleWrite = false;
+ dir1.FailOn(new RandomFailure(this));
+ dir2.FailOn(new RandomFailure(this));
+ dir1.FailOnOpenInput = false;
+ dir2.FailOnOpenInput = false;
+
+ // We throw exceptions in deleteFile, which creates
+ // leftover files:
+ dir1.AssertNoUnrefencedFilesOnClose = false;
+ dir2.AssertNoUnrefencedFilesOnClose = false;
+
+ InitIndex(dir1);
+ InitIndex(dir2);
+
+ TimedThread[] threads = new TimedThread[3];
+ int numThread = 0;
+
+ IndexerThread indexerThread = new IndexerThread(this, this, dir1, dir2, scheduler1, scheduler2, threads);
+ threads[numThread++] = indexerThread;
+ indexerThread.Start();
+
+ SearcherThread searcherThread1 = new SearcherThread(this, dir1, dir2, threads);
+ threads[numThread++] = searcherThread1;
+ searcherThread1.Start();
+
+ SearcherThread searcherThread2 = new SearcherThread(this, dir1, dir2, threads);
+ threads[numThread++] = searcherThread2;
+ searcherThread2.Start();
+
+ for (int i = 0; i < numThread; i++)
+ {
+ threads[i].Join();
+ }
+
+ for (int i = 0; i < numThread; i++)
+ {
+ Assert.IsTrue(!threads[i].Failed);
+ }
+ dir1.Dispose();
+ dir2.Dispose();
+
+ Console.WriteLine("End test");
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestTryDelete.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestTryDelete.cs b/src/Lucene.Net.Tests/Index/TestTryDelete.cs
new file mode 100644
index 0000000..a865675
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestTryDelete.cs
@@ -0,0 +1,196 @@
+using System;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using Lucene.Net.Search;
+ using NUnit.Framework;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+ using ReferenceManager = Lucene.Net.Search.ReferenceManager;
+ using SearcherFactory = Lucene.Net.Search.SearcherFactory;
+ using SearcherManager = Lucene.Net.Search.SearcherManager;
+ using Store = Field.Store;
+ using StringField = StringField;
+ using TermQuery = Lucene.Net.Search.TermQuery;
+ using TopDocs = Lucene.Net.Search.TopDocs;
+
+ [TestFixture]
+ public class TestTryDelete : LuceneTestCase
+ {
+ private static IndexWriter GetWriter(Directory directory)
+ {
+ MergePolicy policy = new LogByteSizeMergePolicy();
+ IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()));
+ conf.SetMergePolicy(policy);
+ conf.SetOpenMode(OpenMode.CREATE_OR_APPEND);
+
+ IndexWriter writer = new IndexWriter(directory, conf);
+
+ return writer;
+ }
+
+ private static Directory CreateIndex()
+ {
+ Directory directory = new RAMDirectory();
+
+ IndexWriter writer = GetWriter(directory);
+
+ for (int i = 0; i < 10; i++)
+ {
+ Document doc = new Document();
+ doc.Add(new StringField("foo", Convert.ToString(i), Store.YES));
+ writer.AddDocument(doc);
+ }
+
+ writer.Commit();
+ writer.Dispose();
+
+ return directory;
+ }
+
+ [Test]
+ public virtual void TestTryDeleteDocument()
+ {
+ Directory directory = CreateIndex();
+
+ IndexWriter writer = GetWriter(directory);
+
+ ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory());
+
+ TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
+
+ IndexSearcher searcher = mgr.Acquire();
+
+ TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+ Assert.AreEqual(1, topDocs.TotalHits);
+
+ long result;
+ if (Random().NextBoolean())
+ {
+ IndexReader r = DirectoryReader.Open(writer, true);
+ result = mgrWriter.TryDeleteDocument(r, 0);
+ r.Dispose();
+ }
+ else
+ {
+ result = mgrWriter.TryDeleteDocument(searcher.IndexReader, 0);
+ }
+
+ // The tryDeleteDocument should have succeeded:
+ Assert.IsTrue(result != -1);
+
+ Assert.IsTrue(writer.HasDeletions);
+
+ if (Random().NextBoolean())
+ {
+ writer.Commit();
+ }
+
+ Assert.IsTrue(writer.HasDeletions);
+
+ mgr.MaybeRefresh();
+
+ searcher = mgr.Acquire();
+
+ topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+
+ Assert.AreEqual(0, topDocs.TotalHits);
+ }
+
+ [Test]
+ public virtual void TestTryDeleteDocumentCloseAndReopen()
+ {
+ Directory directory = CreateIndex();
+
+ IndexWriter writer = GetWriter(directory);
+
+ ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory());
+
+ IndexSearcher searcher = mgr.Acquire();
+
+ TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+ Assert.AreEqual(1, topDocs.TotalHits);
+
+ TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
+ long result = mgrWriter.TryDeleteDocument(DirectoryReader.Open(writer, true), 0);
+
+ Assert.AreEqual(1, result);
+
+ writer.Commit();
+
+ Assert.IsTrue(writer.HasDeletions);
+
+ mgr.MaybeRefresh();
+
+ searcher = mgr.Acquire();
+
+ topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+
+ Assert.AreEqual(0, topDocs.TotalHits);
+
+ writer.Dispose();
+
+ searcher = new IndexSearcher(DirectoryReader.Open(directory));
+
+ topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+
+ Assert.AreEqual(0, topDocs.TotalHits);
+ }
+
+ [Test]
+ public virtual void TestDeleteDocuments()
+ {
+ Directory directory = CreateIndex();
+
+ IndexWriter writer = GetWriter(directory);
+
+ ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer, true, new SearcherFactory());
+
+ IndexSearcher searcher = mgr.Acquire();
+
+ TopDocs topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+ Assert.AreEqual(1, topDocs.TotalHits);
+
+ TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer);
+ long result = mgrWriter.DeleteDocuments(new TermQuery(new Term("foo", "0")));
+
+ Assert.AreEqual(1, result);
+
+ // writer.Commit();
+
+ Assert.IsTrue(writer.HasDeletions);
+
+ mgr.MaybeRefresh();
+
+ searcher = mgr.Acquire();
+
+ topDocs = searcher.Search(new TermQuery(new Term("foo", "0")), 100);
+
+ Assert.AreEqual(0, topDocs.TotalHits);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestTwoPhaseCommitTool.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestTwoPhaseCommitTool.cs b/src/Lucene.Net.Tests/Index/TestTwoPhaseCommitTool.cs
new file mode 100644
index 0000000..58c6f3a
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestTwoPhaseCommitTool.cs
@@ -0,0 +1,178 @@
+using System;
+using System.Collections.Generic;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using NUnit.Framework;
+ using System.IO;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ [TestFixture]
+ public class TestTwoPhaseCommitTool : LuceneTestCase
+ {
+ private class TwoPhaseCommitImpl : ITwoPhaseCommit
+ {
+ internal static bool CommitCalled = false;
+ internal readonly bool FailOnPrepare;
+ internal readonly bool FailOnCommit;
+ internal readonly bool FailOnRollback;
+ internal bool RollbackCalled = false;
+ internal IDictionary<string, string> PrepareCommitData = null;
+ internal IDictionary<string, string> CommitData = null;
+
+ public TwoPhaseCommitImpl(bool failOnPrepare, bool failOnCommit, bool failOnRollback)
+ {
+ this.FailOnPrepare = failOnPrepare;
+ this.FailOnCommit = failOnCommit;
+ this.FailOnRollback = failOnRollback;
+ }
+
+ public void PrepareCommit()
+ {
+ PrepareCommit(null);
+ }
+
+ public virtual void PrepareCommit(IDictionary<string, string> commitData)
+ {
+ this.PrepareCommitData = commitData;
+ Assert.IsFalse(CommitCalled, "commit should not have been called before all prepareCommit were");
+ if (FailOnPrepare)
+ {
+ throw new IOException("failOnPrepare");
+ }
+ }
+
+ public void Commit()
+ {
+ Commit(null);
+ }
+
+ public virtual void Commit(IDictionary<string, string> commitData)
+ {
+ this.CommitData = commitData;
+ CommitCalled = true;
+ if (FailOnCommit)
+ {
+ throw new Exception("failOnCommit");
+ }
+ }
+
+ public void Rollback()
+ {
+ RollbackCalled = true;
+ if (FailOnRollback)
+ {
+ throw new Exception("failOnRollback");
+ }
+ }
+ }
+
+ [SetUp]
+ public override void SetUp()
+ {
+ base.SetUp();
+ TwoPhaseCommitImpl.CommitCalled = false; // reset count before every test
+ }
+
+ [Test]
+ public virtual void TestPrepareThenCommit()
+ {
+ // tests that prepareCommit() is called on all objects before commit()
+ TwoPhaseCommitImpl[] objects = new TwoPhaseCommitImpl[2];
+ for (int i = 0; i < objects.Length; i++)
+ {
+ objects[i] = new TwoPhaseCommitImpl(false, false, false);
+ }
+
+ // following call will fail if commit() is called before all prepare() were
+ TwoPhaseCommitTool.Execute(objects);
+ }
+
+ [Test]
+ public virtual void TestRollback()
+ {
+ // tests that rollback is called if failure occurs at any stage
+ int numObjects = Random().Next(8) + 3; // between [3, 10]
+ TwoPhaseCommitImpl[] objects = new TwoPhaseCommitImpl[numObjects];
+ for (int i = 0; i < objects.Length; i++)
+ {
+ bool failOnPrepare = Random().NextBoolean();
+ // we should not hit failures on commit usually
+ bool failOnCommit = Random().NextDouble() < 0.05;
+ bool railOnRollback = Random().NextBoolean();
+ objects[i] = new TwoPhaseCommitImpl(failOnPrepare, failOnCommit, railOnRollback);
+ }
+
+ bool anyFailure = false;
+ try
+ {
+ TwoPhaseCommitTool.Execute(objects);
+ }
+#pragma warning disable 168
+ catch (Exception t)
+#pragma warning restore 168
+ {
+ anyFailure = true;
+ }
+
+ if (anyFailure)
+ {
+ // if any failure happened, ensure that rollback was called on all.
+ foreach (TwoPhaseCommitImpl tpc in objects)
+ {
+ Assert.IsTrue(tpc.RollbackCalled, "rollback was not called while a failure occurred during the 2-phase commit");
+ }
+ }
+ }
+
+ [Test]
+ public virtual void TestNullTPCs()
+ {
+ int numObjects = Random().Next(4) + 3; // between [3, 6]
+ ITwoPhaseCommit[] tpcs = new ITwoPhaseCommit[numObjects];
+ bool setNull = false;
+ for (int i = 0; i < tpcs.Length; i++)
+ {
+ bool isNull = Random().NextDouble() < 0.3;
+ if (isNull)
+ {
+ setNull = true;
+ tpcs[i] = null;
+ }
+ else
+ {
+ tpcs[i] = new TwoPhaseCommitImpl(false, false, false);
+ }
+ }
+
+ if (!setNull)
+ {
+ // none of the TPCs were picked to be null, pick one at random
+ int idx = Random().Next(numObjects);
+ tpcs[idx] = null;
+ }
+
+ // following call would fail if TPCTool won't handle null TPCs properly
+ TwoPhaseCommitTool.Execute(tpcs);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestUniqueTermCount.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestUniqueTermCount.cs b/src/Lucene.Net.Tests/Index/TestUniqueTermCount.cs
new file mode 100644
index 0000000..46f4722
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestUniqueTermCount.cs
@@ -0,0 +1,122 @@
+\ufeffusing Lucene.Net.Analysis;
+using Lucene.Net.Documents;
+using Lucene.Net.Search;
+using Lucene.Net.Search.Similarities;
+using Lucene.Net.Store;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Lucene.Net.Index
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Tests the uniqueTermCount statistic in FieldInvertState
+ /// </summary>
+ public class TestUniqueTermCount : LuceneTestCase
+ {
+ Directory dir;
+ IndexReader reader;
+ /* expected uniqueTermCount values for our documents */
+ List<int> expected = new List<int>();
+
+ public override void SetUp()
+ {
+ base.SetUp();
+ dir = NewDirectory();
+ MockAnalyzer analyzer = new MockAnalyzer(Random(), MockTokenizer.SIMPLE, true);
+ IndexWriterConfig config = NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
+ config.SetMergePolicy(NewLogMergePolicy());
+ config.SetSimilarity(new TestSimilarity());
+ RandomIndexWriter writer = new RandomIndexWriter(Random(), dir, config);
+ Document doc = new Document();
+ Field foo = NewTextField("foo", "", Field.Store.NO);
+ doc.Add(foo);
+ for (int i = 0; i < 100; i++)
+ {
+ foo.SetStringValue(AddValue());
+ writer.AddDocument(doc);
+ }
+ reader = writer.Reader;
+ writer.Dispose();
+ }
+
+ public override void TearDown()
+ {
+ reader.Dispose();
+ dir.Dispose();
+ base.TearDown();
+ }
+
+ [Test]
+ public void Test()
+ {
+ NumericDocValues fooNorms = MultiDocValues.GetNormValues(reader, "foo");
+ assertNotNull(fooNorms);
+ for (int i = 0; i < reader.MaxDoc; i++)
+ {
+ assertEquals(expected[i], fooNorms.Get(i));
+ }
+ }
+
+ /**
+ * Makes a bunch of single-char tokens (the max # unique terms will at most be 26).
+ * puts the # unique terms into expected, to be checked against the norm.
+ */
+ private string AddValue()
+ {
+ StringBuilder sb = new StringBuilder();
+ HashSet<string> terms = new HashSet<string>();
+ int num = TestUtil.NextInt(Random(), 0, 255);
+ for (int i = 0; i < num; i++)
+ {
+ sb.append(' ');
+ char term = (char)TestUtil.NextInt(Random(), 'a', 'z');
+ sb.append(term);
+ terms.add("" + term);
+ }
+ expected.Add(terms.size());
+ return sb.toString();
+ }
+
+ /**
+ * Simple similarity that encodes maxTermFrequency directly
+ */
+ internal class TestSimilarity : Similarity
+ {
+
+ public override long ComputeNorm(FieldInvertState state)
+ {
+ return state.UniqueTermCount;
+ }
+
+ public override SimWeight ComputeWeight(float queryBoost, CollectionStatistics collectionStats, params TermStatistics[] termStats)
+ {
+ throw new InvalidOperationException();
+ }
+
+ public override SimScorer GetSimScorer(SimWeight weight, AtomicReaderContext context)
+ {
+ throw new InvalidOperationException();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/bogus24.upgraded.to.36.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/bogus24.upgraded.to.36.zip b/src/Lucene.Net.Tests/Index/bogus24.upgraded.to.36.zip
new file mode 100644
index 0000000..52a09f9
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/bogus24.upgraded.to.36.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.30.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.30.cfs.zip b/src/Lucene.Net.Tests/Index/index.30.cfs.zip
new file mode 100644
index 0000000..d5978c8
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.30.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.30.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.30.nocfs.zip b/src/Lucene.Net.Tests/Index/index.30.nocfs.zip
new file mode 100644
index 0000000..28cd83b
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.30.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.31.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.31.cfs.zip b/src/Lucene.Net.Tests/Index/index.31.cfs.zip
new file mode 100644
index 0000000..8f123a7
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.31.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.31.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.31.nocfs.zip b/src/Lucene.Net.Tests/Index/index.31.nocfs.zip
new file mode 100644
index 0000000..21434e1
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.31.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.31.optimized.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.31.optimized.cfs.zip b/src/Lucene.Net.Tests/Index/index.31.optimized.cfs.zip
new file mode 100644
index 0000000..200c710
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.31.optimized.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.31.optimized.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.31.optimized.nocfs.zip b/src/Lucene.Net.Tests/Index/index.31.optimized.nocfs.zip
new file mode 100644
index 0000000..9a158f1
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.31.optimized.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.32.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.32.cfs.zip b/src/Lucene.Net.Tests/Index/index.32.cfs.zip
new file mode 100644
index 0000000..eff3153
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.32.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.32.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.32.nocfs.zip b/src/Lucene.Net.Tests/Index/index.32.nocfs.zip
new file mode 100644
index 0000000..0b345da
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.32.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.34.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.34.cfs.zip b/src/Lucene.Net.Tests/Index/index.34.cfs.zip
new file mode 100644
index 0000000..257e9d8
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.34.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.34.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.34.nocfs.zip b/src/Lucene.Net.Tests/Index/index.34.nocfs.zip
new file mode 100644
index 0000000..935d6a1
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.34.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.36.surrogates.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.36.surrogates.zip b/src/Lucene.Net.Tests/Index/index.36.surrogates.zip
new file mode 100644
index 0000000..6bd7f20
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.36.surrogates.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.40.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.40.cfs.zip b/src/Lucene.Net.Tests/Index/index.40.cfs.zip
new file mode 100644
index 0000000..4974749
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.40.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.40.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.40.nocfs.zip b/src/Lucene.Net.Tests/Index/index.40.nocfs.zip
new file mode 100644
index 0000000..9699080
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.40.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.40.optimized.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.40.optimized.cfs.zip b/src/Lucene.Net.Tests/Index/index.40.optimized.cfs.zip
new file mode 100644
index 0000000..209c436
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.40.optimized.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.40.optimized.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.40.optimized.nocfs.zip b/src/Lucene.Net.Tests/Index/index.40.optimized.nocfs.zip
new file mode 100644
index 0000000..0eaffd0
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.40.optimized.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.41.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.41.cfs.zip b/src/Lucene.Net.Tests/Index/index.41.cfs.zip
new file mode 100644
index 0000000..da2745e
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.41.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.41.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.41.nocfs.zip b/src/Lucene.Net.Tests/Index/index.41.nocfs.zip
new file mode 100644
index 0000000..c056bcb
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.41.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.42.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.42.cfs.zip b/src/Lucene.Net.Tests/Index/index.42.cfs.zip
new file mode 100644
index 0000000..5945fe5
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.42.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.42.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.42.nocfs.zip b/src/Lucene.Net.Tests/Index/index.42.nocfs.zip
new file mode 100644
index 0000000..11de1f1
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.42.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.45.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.45.cfs.zip b/src/Lucene.Net.Tests/Index/index.45.cfs.zip
new file mode 100644
index 0000000..10a8a1a
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.45.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.45.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.45.nocfs.zip b/src/Lucene.Net.Tests/Index/index.45.nocfs.zip
new file mode 100644
index 0000000..7825e2a
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.45.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.461.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.461.cfs.zip b/src/Lucene.Net.Tests/Index/index.461.cfs.zip
new file mode 100644
index 0000000..8f18185
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.461.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/index.461.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/index.461.nocfs.zip b/src/Lucene.Net.Tests/Index/index.461.nocfs.zip
new file mode 100644
index 0000000..cf0173c
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/index.461.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/moreterms.40.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/moreterms.40.zip b/src/Lucene.Net.Tests/Index/moreterms.40.zip
new file mode 100644
index 0000000..53ad7ce
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/moreterms.40.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.19.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.19.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.19.cfs.zip
new file mode 100644
index 0000000..4fd9b32
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.19.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.19.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.19.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.19.nocfs.zip
new file mode 100644
index 0000000..e0d9142
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.19.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.20.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.20.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.20.cfs.zip
new file mode 100644
index 0000000..4b931ae
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.20.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.20.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.20.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.20.nocfs.zip
new file mode 100644
index 0000000..1275cdf
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.20.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.21.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.21.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.21.cfs.zip
new file mode 100644
index 0000000..473c138
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.21.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.21.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.21.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.21.nocfs.zip
new file mode 100644
index 0000000..d0582d0
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.21.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.22.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.22.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.22.cfs.zip
new file mode 100644
index 0000000..1236307
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.22.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.22.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.22.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.22.nocfs.zip
new file mode 100644
index 0000000..216ddf3
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.22.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.23.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.23.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.23.cfs.zip
new file mode 100644
index 0000000..b5fdeef
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.23.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.23.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.23.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.23.nocfs.zip
new file mode 100644
index 0000000..9137ae6
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.23.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.24.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.24.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.24.cfs.zip
new file mode 100644
index 0000000..2c666a9
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.24.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.24.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.24.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.24.nocfs.zip
new file mode 100644
index 0000000..c223875
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.24.nocfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.29.cfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.29.cfs.zip b/src/Lucene.Net.Tests/Index/unsupported.29.cfs.zip
new file mode 100644
index 0000000..c694c78
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.29.cfs.zip differ
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/unsupported.29.nocfs.zip
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/unsupported.29.nocfs.zip b/src/Lucene.Net.Tests/Index/unsupported.29.nocfs.zip
new file mode 100644
index 0000000..298cab7
Binary files /dev/null and b/src/Lucene.Net.Tests/Index/unsupported.29.nocfs.zip differ