You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/02/26 23:37:10 UTC
[22/72] [abbrv] [partial] lucenenet git commit: Lucene.Net.Tests:
Removed \core directory and put its contents in root directory
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestOmitPositions.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestOmitPositions.cs b/src/Lucene.Net.Tests/Index/TestOmitPositions.cs
new file mode 100644
index 0000000..ff8ae7d
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestOmitPositions.cs
@@ -0,0 +1,294 @@
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using NUnit.Framework;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using Analyzer = Lucene.Net.Analysis.Analyzer;
+ using BytesRef = Lucene.Net.Util.BytesRef;
+ using Directory = Lucene.Net.Store.Directory;
+ using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator;
+ using Document = Documents.Document;
+ using Field = Field;
+ using FieldType = FieldType;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+ using TextField = TextField;
+
+ ///
+ /// <summary>
+ /// @lucene.experimental
+ /// </summary>
+ [TestFixture]
+ public class TestOmitPositions : LuceneTestCase
+ {
+ [Test]
+ public virtual void TestBasic()
+ {
+ Directory dir = NewDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
+ Document doc = new Document();
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.IndexOptions = IndexOptions.DOCS_AND_FREQS;
+ Field f = NewField("foo", "this is a test test", ft);
+ doc.Add(f);
+ for (int i = 0; i < 100; i++)
+ {
+ w.AddDocument(doc);
+ }
+
+ IndexReader reader = w.Reader;
+ w.Dispose();
+
+ Assert.IsNull(MultiFields.GetTermPositionsEnum(reader, null, "foo", new BytesRef("test")));
+
+ DocsEnum de = TestUtil.Docs(Random(), reader, "foo", new BytesRef("test"), null, null, DocsEnum.FLAG_FREQS);
+ while (de.NextDoc() != DocIdSetIterator.NO_MORE_DOCS)
+ {
+ Assert.AreEqual(2, de.Freq);
+ }
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ // Tests whether the DocumentWriter correctly enable the
+ // omitTermFreqAndPositions bit in the FieldInfo
+ [Test]
+ public virtual void TestPositions()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
+ Document d = new Document();
+
+ // f1,f2,f3: docs only
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.IndexOptions = IndexOptions.DOCS_ONLY;
+
+ Field f1 = NewField("f1", "this field has docs only", ft);
+ d.Add(f1);
+
+ Field f2 = NewField("f2", "this field has docs only", ft);
+ d.Add(f2);
+
+ Field f3 = NewField("f3", "this field has docs only", ft);
+ d.Add(f3);
+
+ FieldType ft2 = new FieldType(TextField.TYPE_NOT_STORED);
+ ft2.IndexOptions = IndexOptions.DOCS_AND_FREQS;
+
+ // f4,f5,f6 docs and freqs
+ Field f4 = NewField("f4", "this field has docs and freqs", ft2);
+ d.Add(f4);
+
+ Field f5 = NewField("f5", "this field has docs and freqs", ft2);
+ d.Add(f5);
+
+ Field f6 = NewField("f6", "this field has docs and freqs", ft2);
+ d.Add(f6);
+
+ FieldType ft3 = new FieldType(TextField.TYPE_NOT_STORED);
+ ft3.IndexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
+
+ // f7,f8,f9 docs/freqs/positions
+ Field f7 = NewField("f7", "this field has docs and freqs and positions", ft3);
+ d.Add(f7);
+
+ Field f8 = NewField("f8", "this field has docs and freqs and positions", ft3);
+ d.Add(f8);
+
+ Field f9 = NewField("f9", "this field has docs and freqs and positions", ft3);
+ d.Add(f9);
+
+ writer.AddDocument(d);
+ writer.ForceMerge(1);
+
+ // now we add another document which has docs-only for f1, f4, f7, docs/freqs for f2, f5, f8,
+ // and docs/freqs/positions for f3, f6, f9
+ d = new Document();
+
+ // f1,f4,f7: docs only
+ f1 = NewField("f1", "this field has docs only", ft);
+ d.Add(f1);
+
+ f4 = NewField("f4", "this field has docs only", ft);
+ d.Add(f4);
+
+ f7 = NewField("f7", "this field has docs only", ft);
+ d.Add(f7);
+
+ // f2, f5, f8: docs and freqs
+ f2 = NewField("f2", "this field has docs and freqs", ft2);
+ d.Add(f2);
+
+ f5 = NewField("f5", "this field has docs and freqs", ft2);
+ d.Add(f5);
+
+ f8 = NewField("f8", "this field has docs and freqs", ft2);
+ d.Add(f8);
+
+ // f3, f6, f9: docs and freqs and positions
+ f3 = NewField("f3", "this field has docs and freqs and positions", ft3);
+ d.Add(f3);
+
+ f6 = NewField("f6", "this field has docs and freqs and positions", ft3);
+ d.Add(f6);
+
+ f9 = NewField("f9", "this field has docs and freqs and positions", ft3);
+ d.Add(f9);
+
+ writer.AddDocument(d);
+
+ // force merge
+ writer.ForceMerge(1);
+ // flush
+ writer.Dispose();
+
+ SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram));
+ FieldInfos fi = reader.FieldInfos;
+ // docs + docs = docs
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f1").IndexOptions);
+ // docs + docs/freqs = docs
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f2").IndexOptions);
+ // docs + docs/freqs/pos = docs
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f3").IndexOptions);
+ // docs/freqs + docs = docs
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f4").IndexOptions);
+ // docs/freqs + docs/freqs = docs/freqs
+ Assert.AreEqual(IndexOptions.DOCS_AND_FREQS, fi.FieldInfo("f5").IndexOptions);
+ // docs/freqs + docs/freqs/pos = docs/freqs
+ Assert.AreEqual(IndexOptions.DOCS_AND_FREQS, fi.FieldInfo("f6").IndexOptions);
+ // docs/freqs/pos + docs = docs
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f7").IndexOptions);
+ // docs/freqs/pos + docs/freqs = docs/freqs
+ Assert.AreEqual(IndexOptions.DOCS_AND_FREQS, fi.FieldInfo("f8").IndexOptions);
+ // docs/freqs/pos + docs/freqs/pos = docs/freqs/pos
+ Assert.AreEqual(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.FieldInfo("f9").IndexOptions);
+
+ reader.Dispose();
+ ram.Dispose();
+ }
+
+ private void AssertNoPrx(Directory dir)
+ {
+ string[] files = dir.ListAll();
+ for (int i = 0; i < files.Length; i++)
+ {
+ Assert.IsFalse(files[i].EndsWith(".prx"));
+ Assert.IsFalse(files[i].EndsWith(".pos"));
+ }
+ }
+
+ // Verifies no *.prx exists when all fields omit term positions:
+ [Test]
+ public virtual void TestNoPrxFile()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(3).SetMergePolicy(NewLogMergePolicy()));
+ LogMergePolicy lmp = (LogMergePolicy)writer.Config.MergePolicy;
+ lmp.MergeFactor = 2;
+ lmp.NoCFSRatio = 0.0;
+ Document d = new Document();
+
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.IndexOptions = IndexOptions.DOCS_AND_FREQS;
+ Field f1 = NewField("f1", "this field has term freqs", ft);
+ d.Add(f1);
+
+ for (int i = 0; i < 30; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ writer.Commit();
+
+ AssertNoPrx(ram);
+
+ // now add some documents with positions, and check there is no prox after optimization
+ d = new Document();
+ f1 = NewTextField("f1", "this field has positions", Field.Store.NO);
+ d.Add(f1);
+
+ for (int i = 0; i < 30; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ // force merge
+ writer.ForceMerge(1);
+ // flush
+ writer.Dispose();
+
+ AssertNoPrx(ram);
+ ram.Dispose();
+ }
+
+ /// <summary>
+ /// make sure we downgrade positions and payloads correctly </summary>
+ [Test]
+ public virtual void TestMixing()
+ {
+ // no positions
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.IndexOptions = IndexOptions.DOCS_AND_FREQS;
+
+ Directory dir = NewDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, Similarity, TimeZone);
+
+ for (int i = 0; i < 20; i++)
+ {
+ Document doc = new Document();
+ if (i < 19 && Random().NextBoolean())
+ {
+ for (int j = 0; j < 50; j++)
+ {
+ doc.Add(new TextField("foo", "i have positions", Field.Store.NO));
+ }
+ }
+ else
+ {
+ for (int j = 0; j < 50; j++)
+ {
+ doc.Add(new Field("foo", "i have no positions", ft));
+ }
+ }
+ iw.AddDocument(doc);
+ iw.Commit();
+ }
+
+ if (Random().NextBoolean())
+ {
+ iw.ForceMerge(1);
+ }
+
+ DirectoryReader ir = iw.Reader;
+ FieldInfos fis = MultiFields.GetMergedFieldInfos(ir);
+ Assert.AreEqual(IndexOptions.DOCS_AND_FREQS, fis.FieldInfo("foo").IndexOptions);
+ Assert.IsFalse(fis.FieldInfo("foo").HasPayloads);
+ iw.Dispose();
+ ir.Dispose();
+ dir.Dispose(); // checkindex
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestOmitTf.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestOmitTf.cs b/src/Lucene.Net.Tests/Index/TestOmitTf.cs
new file mode 100644
index 0000000..3286d4b
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestOmitTf.cs
@@ -0,0 +1,588 @@
+using System;
+using System.Text;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using NUnit.Framework;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using Analyzer = Lucene.Net.Analysis.Analyzer;
+ using BooleanQuery = Lucene.Net.Search.BooleanQuery;
+ using BytesRef = Lucene.Net.Util.BytesRef;
+ using CollectionStatistics = Lucene.Net.Search.CollectionStatistics;
+ using ICollector = Lucene.Net.Search.ICollector;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using Explanation = Lucene.Net.Search.Explanation;
+ using Field = Field;
+ using FieldType = FieldType;
+ using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using Occur = Lucene.Net.Search.Occur;
+ using PhraseQuery = Lucene.Net.Search.PhraseQuery;
+ using Scorer = Lucene.Net.Search.Scorer;
+ using TermQuery = Lucene.Net.Search.TermQuery;
+ using TermStatistics = Lucene.Net.Search.TermStatistics;
+ using TextField = TextField;
+ using TFIDFSimilarity = Lucene.Net.Search.Similarities.TFIDFSimilarity;
+
+ [TestFixture]
+ public class TestOmitTf : LuceneTestCase
+ {
+ public class SimpleSimilarity : TFIDFSimilarity
+ {
+ public override float DecodeNormValue(long norm)
+ {
+ return norm;
+ }
+
+ public override long EncodeNormValue(float f)
+ {
+ return (long)f;
+ }
+
+ public override float QueryNorm(float sumOfSquaredWeights)
+ {
+ return 1.0f;
+ }
+
+ public override float Coord(int overlap, int maxOverlap)
+ {
+ return 1.0f;
+ }
+
+ public override float LengthNorm(FieldInvertState state)
+ {
+ return state.Boost;
+ }
+
+ public override float Tf(float freq)
+ {
+ return freq;
+ }
+
+ public override float SloppyFreq(int distance)
+ {
+ return 2.0f;
+ }
+
+ public override float Idf(long docFreq, long numDocs)
+ {
+ return 1.0f;
+ }
+
+ public override Explanation IdfExplain(CollectionStatistics collectionStats, TermStatistics[] termStats)
+ {
+ return new Explanation(1.0f, "Inexplicable");
+ }
+
+ public override float ScorePayload(int doc, int start, int end, BytesRef payload)
+ {
+ return 1.0f;
+ }
+ }
+
+ private static readonly FieldType OmitType = new FieldType(TextField.TYPE_NOT_STORED);
+ private static readonly FieldType NormalType = new FieldType(TextField.TYPE_NOT_STORED);
+
+ static TestOmitTf()
+ {
+ OmitType.IndexOptions = IndexOptions.DOCS_ONLY;
+ }
+
+ // Tests whether the DocumentWriter correctly enable the
+ // omitTermFreqAndPositions bit in the FieldInfo
+ [Test]
+ public virtual void TestOmitTermFreqAndPositions()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
+ Document d = new Document();
+
+ // this field will have Tf
+ Field f1 = NewField("f1", "this field has term freqs", NormalType);
+ d.Add(f1);
+
+ // this field will NOT have Tf
+ Field f2 = NewField("f2", "this field has NO Tf in all docs", OmitType);
+ d.Add(f2);
+
+ writer.AddDocument(d);
+ writer.ForceMerge(1);
+ // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger
+ // keep things constant
+ d = new Document();
+
+ // Reverse
+ f1 = NewField("f1", "this field has term freqs", OmitType);
+ d.Add(f1);
+
+ f2 = NewField("f2", "this field has NO Tf in all docs", NormalType);
+ d.Add(f2);
+
+ writer.AddDocument(d);
+
+ // force merge
+ writer.ForceMerge(1);
+ // flush
+ writer.Dispose();
+
+ SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram));
+ FieldInfos fi = reader.FieldInfos;
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f1").IndexOptions, "OmitTermFreqAndPositions field bit should be set.");
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f2").IndexOptions, "OmitTermFreqAndPositions field bit should be set.");
+
+ reader.Dispose();
+ ram.Dispose();
+ }
+
+ // Tests whether merging of docs that have different
+ // omitTermFreqAndPositions for the same field works
+ [Test]
+ public virtual void TestMixedMerge()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(3).SetMergePolicy(NewLogMergePolicy(2)));
+ Document d = new Document();
+
+ // this field will have Tf
+ Field f1 = NewField("f1", "this field has term freqs", NormalType);
+ d.Add(f1);
+
+ // this field will NOT have Tf
+ Field f2 = NewField("f2", "this field has NO Tf in all docs", OmitType);
+ d.Add(f2);
+
+ for (int i = 0; i < 30; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger
+ // keep things constant
+ d = new Document();
+
+ // Reverese
+ f1 = NewField("f1", "this field has term freqs", OmitType);
+ d.Add(f1);
+
+ f2 = NewField("f2", "this field has NO Tf in all docs", NormalType);
+ d.Add(f2);
+
+ for (int i = 0; i < 30; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ // force merge
+ writer.ForceMerge(1);
+ // flush
+ writer.Dispose();
+
+ SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram));
+ FieldInfos fi = reader.FieldInfos;
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f1").IndexOptions, "OmitTermFreqAndPositions field bit should be set.");
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f2").IndexOptions, "OmitTermFreqAndPositions field bit should be set.");
+
+ reader.Dispose();
+ ram.Dispose();
+ }
+
+ // Make sure first adding docs that do not omitTermFreqAndPositions for
+ // field X, then adding docs that do omitTermFreqAndPositions for that same
+ // field,
+ [Test]
+ public virtual void TestMixedRAM()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(10).SetMergePolicy(NewLogMergePolicy(2)));
+ Document d = new Document();
+
+ // this field will have Tf
+ Field f1 = NewField("f1", "this field has term freqs", NormalType);
+ d.Add(f1);
+
+ // this field will NOT have Tf
+ Field f2 = NewField("f2", "this field has NO Tf in all docs", OmitType);
+ d.Add(f2);
+
+ for (int i = 0; i < 5; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ for (int i = 0; i < 20; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ // force merge
+ writer.ForceMerge(1);
+
+ // flush
+ writer.Dispose();
+
+ SegmentReader reader = GetOnlySegmentReader(DirectoryReader.Open(ram));
+ FieldInfos fi = reader.FieldInfos;
+ Assert.AreEqual(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS, fi.FieldInfo("f1").IndexOptions, "OmitTermFreqAndPositions field bit should not be set.");
+ Assert.AreEqual(IndexOptions.DOCS_ONLY, fi.FieldInfo("f2").IndexOptions, "OmitTermFreqAndPositions field bit should be set.");
+
+ reader.Dispose();
+ ram.Dispose();
+ }
+
+ private void AssertNoPrx(Directory dir)
+ {
+ string[] files = dir.ListAll();
+ for (int i = 0; i < files.Length; i++)
+ {
+ Assert.IsFalse(files[i].EndsWith(".prx"));
+ Assert.IsFalse(files[i].EndsWith(".pos"));
+ }
+ }
+
+ // Verifies no *.prx exists when all fields omit term freq:
+ [Test]
+ public virtual void TestNoPrxFile()
+ {
+ Directory ram = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(ram, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(3).SetMergePolicy(NewLogMergePolicy()));
+ LogMergePolicy lmp = (LogMergePolicy)writer.Config.MergePolicy;
+ lmp.MergeFactor = 2;
+ lmp.NoCFSRatio = 0.0;
+ Document d = new Document();
+
+ Field f1 = NewField("f1", "this field has term freqs", OmitType);
+ d.Add(f1);
+
+ for (int i = 0; i < 30; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ writer.Commit();
+
+ AssertNoPrx(ram);
+
+ // now add some documents with positions, and check
+ // there is no prox after full merge
+ d = new Document();
+ f1 = NewTextField("f1", "this field has positions", Field.Store.NO);
+ d.Add(f1);
+
+ for (int i = 0; i < 30; i++)
+ {
+ writer.AddDocument(d);
+ }
+
+ // force merge
+ writer.ForceMerge(1);
+ // flush
+ writer.Dispose();
+
+ AssertNoPrx(ram);
+ ram.Dispose();
+ }
+
+ // Test scores with one field with Term Freqs and one without, otherwise with equal content
+ [Test]
+ public virtual void TestBasic()
+ {
+ Directory dir = NewDirectory();
+ Analyzer analyzer = new MockAnalyzer(Random());
+ IndexWriter writer = new IndexWriter(dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, analyzer).SetMaxBufferedDocs(2).SetSimilarity(new SimpleSimilarity()).SetMergePolicy(NewLogMergePolicy(2)));
+
+ StringBuilder sb = new StringBuilder(265);
+ string term = "term";
+ for (int i = 0; i < 30; i++)
+ {
+ Document doc = new Document();
+ sb.Append(term).Append(" ");
+ string content = sb.ToString();
+ Field noTf = NewField("noTf", content + (i % 2 == 0 ? "" : " notf"), OmitType);
+ doc.Add(noTf);
+
+ Field tf = NewField("tf", content + (i % 2 == 0 ? " tf" : ""), NormalType);
+ doc.Add(tf);
+
+ writer.AddDocument(doc);
+ //System.out.println(d);
+ }
+
+ writer.ForceMerge(1);
+ // flush
+ writer.Dispose();
+
+ /*
+ * Verify the index
+ */
+ IndexReader reader = DirectoryReader.Open(dir);
+ IndexSearcher searcher = NewSearcher(reader);
+ searcher.Similarity = new SimpleSimilarity();
+
+ Term a = new Term("noTf", term);
+ Term b = new Term("tf", term);
+ Term c = new Term("noTf", "notf");
+ Term d = new Term("tf", "tf");
+ TermQuery q1 = new TermQuery(a);
+ TermQuery q2 = new TermQuery(b);
+ TermQuery q3 = new TermQuery(c);
+ TermQuery q4 = new TermQuery(d);
+
+ PhraseQuery pq = new PhraseQuery();
+ pq.Add(a);
+ pq.Add(c);
+ try
+ {
+ searcher.Search(pq, 10);
+ Assert.Fail("did not hit expected exception");
+ }
+ catch (Exception e)
+ {
+ Exception cause = e;
+ // If the searcher uses an executor service, the IAE is wrapped into other exceptions
+ while (cause.InnerException != null)
+ {
+ cause = cause.InnerException;
+ }
+ if (!(cause is InvalidOperationException))
+ {
+ throw new InvalidOperationException("Expected an IAE", e);
+ } // else OK because positions are not indexed
+ }
+
+ searcher.Search(q1, new CountingHitCollectorAnonymousInnerClassHelper(this));
+ //System.out.println(CountingHitCollector.getCount());
+
+ searcher.Search(q2, new CountingHitCollectorAnonymousInnerClassHelper2(this));
+ //System.out.println(CountingHitCollector.getCount());
+
+ searcher.Search(q3, new CountingHitCollectorAnonymousInnerClassHelper3(this));
+ //System.out.println(CountingHitCollector.getCount());
+
+ searcher.Search(q4, new CountingHitCollectorAnonymousInnerClassHelper4(this));
+ //System.out.println(CountingHitCollector.getCount());
+
+ BooleanQuery bq = new BooleanQuery();
+ bq.Add(q1, Occur.MUST);
+ bq.Add(q4, Occur.MUST);
+
+ searcher.Search(bq, new CountingHitCollectorAnonymousInnerClassHelper5(this));
+ Assert.AreEqual(15, CountingHitCollector.Count);
+
+ reader.Dispose();
+ dir.Dispose();
+ }
+
+ private class CountingHitCollectorAnonymousInnerClassHelper : CountingHitCollector
+ {
+ private readonly TestOmitTf OuterInstance;
+
+ public CountingHitCollectorAnonymousInnerClassHelper(TestOmitTf outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ }
+
+ private Scorer scorer;
+
+ public override sealed void SetScorer(Scorer scorer)
+ {
+ this.scorer = scorer;
+ }
+
+ public override sealed void Collect(int doc)
+ {
+ //System.out.println("Q1: Doc=" + doc + " score=" + score);
+ float score = scorer.GetScore();
+ Assert.IsTrue(score == 1.0f, "got score=" + score);
+ base.Collect(doc);
+ }
+ }
+
+ private class CountingHitCollectorAnonymousInnerClassHelper2 : CountingHitCollector
+ {
+ private readonly TestOmitTf OuterInstance;
+
+ public CountingHitCollectorAnonymousInnerClassHelper2(TestOmitTf outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ }
+
+ private Scorer scorer;
+
+ public override sealed void SetScorer(Scorer scorer)
+ {
+ this.scorer = scorer;
+ }
+
+ public override sealed void Collect(int doc)
+ {
+ //System.out.println("Q2: Doc=" + doc + " score=" + score);
+ float score = scorer.GetScore();
+ Assert.AreEqual(1.0f + doc, score, 0.00001f);
+ base.Collect(doc);
+ }
+ }
+
+ private class CountingHitCollectorAnonymousInnerClassHelper3 : CountingHitCollector
+ {
+ private readonly TestOmitTf OuterInstance;
+
+ public CountingHitCollectorAnonymousInnerClassHelper3(TestOmitTf outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ }
+
+ private Scorer scorer;
+
+ public override sealed void SetScorer(Scorer scorer)
+ {
+ this.scorer = scorer;
+ }
+
+ public override sealed void Collect(int doc)
+ {
+ //System.out.println("Q1: Doc=" + doc + " score=" + score);
+ float score = scorer.GetScore();
+ Assert.IsTrue(score == 1.0f);
+ Assert.IsFalse(doc % 2 == 0);
+ base.Collect(doc);
+ }
+ }
+
+ private class CountingHitCollectorAnonymousInnerClassHelper4 : CountingHitCollector
+ {
+ private readonly TestOmitTf OuterInstance;
+
+ public CountingHitCollectorAnonymousInnerClassHelper4(TestOmitTf outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ }
+
+ private Scorer scorer;
+
+ public override sealed void SetScorer(Scorer scorer)
+ {
+ this.scorer = scorer;
+ }
+
+ public override sealed void Collect(int doc)
+ {
+ float score = scorer.GetScore();
+ //System.out.println("Q1: Doc=" + doc + " score=" + score);
+ Assert.IsTrue(score == 1.0f);
+ Assert.IsTrue(doc % 2 == 0);
+ base.Collect(doc);
+ }
+ }
+
+ private class CountingHitCollectorAnonymousInnerClassHelper5 : CountingHitCollector
+ {
+ private readonly TestOmitTf OuterInstance;
+
+ public CountingHitCollectorAnonymousInnerClassHelper5(TestOmitTf outerInstance)
+ {
+ this.OuterInstance = outerInstance;
+ }
+
+ public override sealed void Collect(int doc)
+ {
+ //System.out.println("BQ: Doc=" + doc + " score=" + score);
+ base.Collect(doc);
+ }
+ }
+
+ public class CountingHitCollector : ICollector
+ {
+ internal static int Count_Renamed = 0;
+ internal static int Sum_Renamed = 0;
+ internal int DocBase = -1;
+
+ internal CountingHitCollector()
+ {
+ Count_Renamed = 0;
+ Sum_Renamed = 0;
+ }
+
+ public virtual void SetScorer(Scorer scorer)
+ {
+ }
+
+ public virtual void Collect(int doc)
+ {
+ Count_Renamed++;
+ Sum_Renamed += doc + DocBase; // use it to avoid any possibility of being merged away
+ }
+
+ public static int Count
+ {
+ get
+ {
+ return Count_Renamed;
+ }
+ }
+
+ public static int Sum
+ {
+ get
+ {
+ return Sum_Renamed;
+ }
+ }
+
+ public virtual void SetNextReader(AtomicReaderContext context)
+ {
+ DocBase = context.DocBase;
+ }
+
+ public virtual bool AcceptsDocsOutOfOrder
+ {
+ get { return true; }
+ }
+ }
+
+ /// <summary>
+ /// test that when freqs are omitted, that totalTermFreq and sumTotalTermFreq are -1 </summary>
+ [Test]
+ public virtual void TestStats()
+ {
+ Directory dir = NewDirectory();
+ RandomIndexWriter iw = new RandomIndexWriter(Random(), dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.IndexOptions = IndexOptions.DOCS_ONLY;
+ ft.Freeze();
+ Field f = NewField("foo", "bar", ft);
+ doc.Add(f);
+ iw.AddDocument(doc);
+ IndexReader ir = iw.Reader;
+ iw.Dispose();
+ Assert.AreEqual(-1, ir.TotalTermFreq(new Term("foo", new BytesRef("bar"))));
+ Assert.AreEqual(-1, ir.GetSumTotalTermFreq("foo"));
+ ir.Dispose();
+ dir.Dispose();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestParallelAtomicReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestParallelAtomicReader.cs b/src/Lucene.Net.Tests/Index/TestParallelAtomicReader.cs
new file mode 100644
index 0000000..c6e896a
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestParallelAtomicReader.cs
@@ -0,0 +1,357 @@
+using System;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using Lucene.Net.Search;
+ using NUnit.Framework;
+ using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using Field = Field;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using Occur = Lucene.Net.Search.Occur;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+
+ [TestFixture]
+ public class TestParallelAtomicReader : LuceneTestCase
+ {
+ private IndexSearcher Parallel_Renamed, Single_Renamed;
+ private Directory Dir, Dir1, Dir2;
+
+ [Test]
+ public virtual void TestQueries()
+ {
+ Single_Renamed = Single(Random());
+ Parallel_Renamed = Parallel(Random());
+
+ QueryTest(new TermQuery(new Term("f1", "v1")));
+ QueryTest(new TermQuery(new Term("f1", "v2")));
+ QueryTest(new TermQuery(new Term("f2", "v1")));
+ QueryTest(new TermQuery(new Term("f2", "v2")));
+ QueryTest(new TermQuery(new Term("f3", "v1")));
+ QueryTest(new TermQuery(new Term("f3", "v2")));
+ QueryTest(new TermQuery(new Term("f4", "v1")));
+ QueryTest(new TermQuery(new Term("f4", "v2")));
+
+ BooleanQuery bq1 = new BooleanQuery();
+ bq1.Add(new TermQuery(new Term("f1", "v1")), Occur.MUST);
+ bq1.Add(new TermQuery(new Term("f4", "v1")), Occur.MUST);
+ QueryTest(bq1);
+
+ Single_Renamed.IndexReader.Dispose();
+ Single_Renamed = null;
+ Parallel_Renamed.IndexReader.Dispose();
+ Parallel_Renamed = null;
+ Dir.Dispose();
+ Dir = null;
+ Dir1.Dispose();
+ Dir1 = null;
+ Dir2.Dispose();
+ Dir2 = null;
+ }
+
+ [Test]
+ public virtual void TestFieldNames()
+ {
+ Directory dir1 = GetDir1(Random());
+ Directory dir2 = GetDir2(Random());
+ ParallelAtomicReader pr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1)), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2)));
+ FieldInfos fieldInfos = pr.FieldInfos;
+ Assert.AreEqual(4, fieldInfos.Count);
+ Assert.IsNotNull(fieldInfos.FieldInfo("f1"));
+ Assert.IsNotNull(fieldInfos.FieldInfo("f2"));
+ Assert.IsNotNull(fieldInfos.FieldInfo("f3"));
+ Assert.IsNotNull(fieldInfos.FieldInfo("f4"));
+ pr.Dispose();
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void TestRefCounts1()
+ {
+ Directory dir1 = GetDir1(Random());
+ Directory dir2 = GetDir2(Random());
+ AtomicReader ir1, ir2;
+ // close subreaders, ParallelReader will not change refCounts, but close on its own close
+ ParallelAtomicReader pr = new ParallelAtomicReader(ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1)), ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2)));
+
+ // check RefCounts
+ Assert.AreEqual(1, ir1.RefCount);
+ Assert.AreEqual(1, ir2.RefCount);
+ pr.Dispose();
+ Assert.AreEqual(0, ir1.RefCount);
+ Assert.AreEqual(0, ir2.RefCount);
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void TestRefCounts2()
+ {
+ Directory dir1 = GetDir1(Random());
+ Directory dir2 = GetDir2(Random());
+ AtomicReader ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1));
+ AtomicReader ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2));
+ // don't close subreaders, so ParallelReader will increment refcounts
+ ParallelAtomicReader pr = new ParallelAtomicReader(false, ir1, ir2);
+ // check RefCounts
+ Assert.AreEqual(2, ir1.RefCount);
+ Assert.AreEqual(2, ir2.RefCount);
+ pr.Dispose();
+ Assert.AreEqual(1, ir1.RefCount);
+ Assert.AreEqual(1, ir2.RefCount);
+ ir1.Dispose();
+ ir2.Dispose();
+ Assert.AreEqual(0, ir1.RefCount);
+ Assert.AreEqual(0, ir2.RefCount);
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void TestCloseInnerReader()
+ {
+ Directory dir1 = GetDir1(Random());
+ AtomicReader ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1));
+
+ // with overlapping
+ ParallelAtomicReader pr = new ParallelAtomicReader(true, new AtomicReader[] { ir1 }, new AtomicReader[] { ir1 });
+
+ ir1.Dispose();
+
+ try
+ {
+ pr.Document(0);
+ Assert.Fail("ParallelAtomicReader should be already closed because inner reader was closed!");
+ }
+#pragma warning disable 168
+ catch (AlreadyClosedException e)
+#pragma warning restore 168
+ {
+ // pass
+ }
+
+ // noop:
+ pr.Dispose();
+ dir1.Dispose();
+ }
+
+ [Test]
+ public virtual void TestIncompatibleIndexes()
+ {
+ // two documents:
+ Directory dir1 = GetDir1(Random());
+
+ // one document only:
+ Directory dir2 = NewDirectory();
+ IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document d3 = new Document();
+
+ d3.Add(NewTextField("f3", "v1", Field.Store.YES));
+ w2.AddDocument(d3);
+ w2.Dispose();
+
+ AtomicReader ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1));
+ AtomicReader ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2));
+
+ try
+ {
+ new ParallelAtomicReader(ir1, ir2);
+ Assert.Fail("didn't get exptected exception: indexes don't have same number of documents");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // expected exception
+ }
+
+ try
+ {
+ new ParallelAtomicReader(Random().NextBoolean(), new AtomicReader[] { ir1, ir2 }, new AtomicReader[] { ir1, ir2 });
+ Assert.Fail("didn't get expected exception: indexes don't have same number of documents");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // expected exception
+ }
+ // check RefCounts
+ Assert.AreEqual(1, ir1.RefCount);
+ Assert.AreEqual(1, ir2.RefCount);
+ ir1.Dispose();
+ ir2.Dispose();
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void TestIgnoreStoredFields()
+ {
+ Directory dir1 = GetDir1(Random());
+ Directory dir2 = GetDir2(Random());
+ AtomicReader ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1));
+ AtomicReader ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir2));
+
+ // with overlapping
+ ParallelAtomicReader pr = new ParallelAtomicReader(false, new AtomicReader[] { ir1, ir2 }, new AtomicReader[] { ir1 });
+ Assert.AreEqual("v1", pr.Document(0).Get("f1"));
+ Assert.AreEqual("v1", pr.Document(0).Get("f2"));
+ Assert.IsNull(pr.Document(0).Get("f3"));
+ Assert.IsNull(pr.Document(0).Get("f4"));
+ // check that fields are there
+ Assert.IsNotNull(pr.Terms("f1"));
+ Assert.IsNotNull(pr.Terms("f2"));
+ Assert.IsNotNull(pr.Terms("f3"));
+ Assert.IsNotNull(pr.Terms("f4"));
+ pr.Dispose();
+
+ // no stored fields at all
+ pr = new ParallelAtomicReader(false, new AtomicReader[] { ir2 }, new AtomicReader[0]);
+ Assert.IsNull(pr.Document(0).Get("f1"));
+ Assert.IsNull(pr.Document(0).Get("f2"));
+ Assert.IsNull(pr.Document(0).Get("f3"));
+ Assert.IsNull(pr.Document(0).Get("f4"));
+ // check that fields are there
+ Assert.IsNull(pr.Terms("f1"));
+ Assert.IsNull(pr.Terms("f2"));
+ Assert.IsNotNull(pr.Terms("f3"));
+ Assert.IsNotNull(pr.Terms("f4"));
+ pr.Dispose();
+
+ // without overlapping
+ pr = new ParallelAtomicReader(true, new AtomicReader[] { ir2 }, new AtomicReader[] { ir1 });
+ Assert.AreEqual("v1", pr.Document(0).Get("f1"));
+ Assert.AreEqual("v1", pr.Document(0).Get("f2"));
+ Assert.IsNull(pr.Document(0).Get("f3"));
+ Assert.IsNull(pr.Document(0).Get("f4"));
+ // check that fields are there
+ Assert.IsNull(pr.Terms("f1"));
+ Assert.IsNull(pr.Terms("f2"));
+ Assert.IsNotNull(pr.Terms("f3"));
+ Assert.IsNotNull(pr.Terms("f4"));
+ pr.Dispose();
+
+ // no main readers
+ try
+ {
+ new ParallelAtomicReader(true, new AtomicReader[0], new AtomicReader[] { ir1 });
+ Assert.Fail("didn't get expected exception: need a non-empty main-reader array");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException iae)
+#pragma warning restore 168
+ {
+ // pass
+ }
+
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ private void QueryTest(Query query)
+ {
+ ScoreDoc[] parallelHits = Parallel_Renamed.Search(query, null, 1000).ScoreDocs;
+ ScoreDoc[] singleHits = Single_Renamed.Search(query, null, 1000).ScoreDocs;
+ Assert.AreEqual(parallelHits.Length, singleHits.Length);
+ for (int i = 0; i < parallelHits.Length; i++)
+ {
+ Assert.AreEqual(parallelHits[i].Score, singleHits[i].Score, 0.001f);
+ Document docParallel = Parallel_Renamed.Doc(parallelHits[i].Doc);
+ Document docSingle = Single_Renamed.Doc(singleHits[i].Doc);
+ Assert.AreEqual(docParallel.Get("f1"), docSingle.Get("f1"));
+ Assert.AreEqual(docParallel.Get("f2"), docSingle.Get("f2"));
+ Assert.AreEqual(docParallel.Get("f3"), docSingle.Get("f3"));
+ Assert.AreEqual(docParallel.Get("f4"), docSingle.Get("f4"));
+ }
+ }
+
+ // Fields 1-4 indexed together:
+ private IndexSearcher Single(Random random)
+ {
+ Dir = NewDirectory();
+ IndexWriter w = new IndexWriter(Dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ Document d1 = new Document();
+ d1.Add(NewTextField("f1", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f2", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f3", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f4", "v1", Field.Store.YES));
+ w.AddDocument(d1);
+ Document d2 = new Document();
+ d2.Add(NewTextField("f1", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f2", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f3", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f4", "v2", Field.Store.YES));
+ w.AddDocument(d2);
+ w.Dispose();
+
+ DirectoryReader ir = DirectoryReader.Open(Dir);
+ return NewSearcher(ir);
+ }
+
+ // Fields 1 & 2 in one index, 3 & 4 in other, with ParallelReader:
+ private IndexSearcher Parallel(Random random)
+ {
+ Dir1 = GetDir1(random);
+ Dir2 = GetDir2(random);
+ ParallelAtomicReader pr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(Dir1)), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(Dir2)));
+ TestUtil.CheckReader(pr);
+ return NewSearcher(pr);
+ }
+
+ private Directory GetDir1(Random random)
+ {
+ Directory dir1 = NewDirectory();
+ IndexWriter w1 = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ Document d1 = new Document();
+ d1.Add(NewTextField("f1", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f2", "v1", Field.Store.YES));
+ w1.AddDocument(d1);
+ Document d2 = new Document();
+ d2.Add(NewTextField("f1", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f2", "v2", Field.Store.YES));
+ w1.AddDocument(d2);
+ w1.Dispose();
+ return dir1;
+ }
+
+ private Directory GetDir2(Random random)
+ {
+ Directory dir2 = NewDirectory();
+ IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ Document d3 = new Document();
+ d3.Add(NewTextField("f3", "v1", Field.Store.YES));
+ d3.Add(NewTextField("f4", "v1", Field.Store.YES));
+ w2.AddDocument(d3);
+ Document d4 = new Document();
+ d4.Add(NewTextField("f3", "v2", Field.Store.YES));
+ d4.Add(NewTextField("f4", "v2", Field.Store.YES));
+ w2.AddDocument(d4);
+ w2.Dispose();
+ return dir2;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestParallelCompositeReader.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestParallelCompositeReader.cs b/src/Lucene.Net.Tests/Index/TestParallelCompositeReader.cs
new file mode 100644
index 0000000..8b7da0e
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestParallelCompositeReader.cs
@@ -0,0 +1,666 @@
+using System;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using Lucene.Net.Randomized.Generators;
+ using Lucene.Net.Search;
+ using NUnit.Framework;
+ using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using Field = Field;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using Occur = Lucene.Net.Search.Occur;
+ using IReaderClosedListener = Lucene.Net.Index.IndexReader.IReaderClosedListener;
+
+ [TestFixture]
+ public class TestParallelCompositeReader : LuceneTestCase
+ {
+ private IndexSearcher Parallel_Renamed, Single_Renamed;
+ private Directory Dir, Dir1, Dir2;
+
+ [Test]
+ public virtual void TestQueries()
+ {
+ Single_Renamed = Single(Random(), false);
+ Parallel_Renamed = Parallel(Random(), false);
+
+ Queries();
+
+ Single_Renamed.IndexReader.Dispose();
+ Single_Renamed = null;
+ Parallel_Renamed.IndexReader.Dispose();
+ Parallel_Renamed = null;
+ Dir.Dispose();
+ Dir = null;
+ Dir1.Dispose();
+ Dir1 = null;
+ Dir2.Dispose();
+ Dir2 = null;
+ }
+
+ [Test]
+ public virtual void TestQueriesCompositeComposite()
+ {
+ Single_Renamed = Single(Random(), true);
+ Parallel_Renamed = Parallel(Random(), true);
+
+ Queries();
+
+ Single_Renamed.IndexReader.Dispose();
+ Single_Renamed = null;
+ Parallel_Renamed.IndexReader.Dispose();
+ Parallel_Renamed = null;
+ Dir.Dispose();
+ Dir = null;
+ Dir1.Dispose();
+ Dir1 = null;
+ Dir2.Dispose();
+ Dir2 = null;
+ }
+
+ private void Queries()
+ {
+ QueryTest(new TermQuery(new Term("f1", "v1")));
+ QueryTest(new TermQuery(new Term("f1", "v2")));
+ QueryTest(new TermQuery(new Term("f2", "v1")));
+ QueryTest(new TermQuery(new Term("f2", "v2")));
+ QueryTest(new TermQuery(new Term("f3", "v1")));
+ QueryTest(new TermQuery(new Term("f3", "v2")));
+ QueryTest(new TermQuery(new Term("f4", "v1")));
+ QueryTest(new TermQuery(new Term("f4", "v2")));
+
+ BooleanQuery bq1 = new BooleanQuery();
+ bq1.Add(new TermQuery(new Term("f1", "v1")), Occur.MUST);
+ bq1.Add(new TermQuery(new Term("f4", "v1")), Occur.MUST);
+ QueryTest(bq1);
+ }
+
+ [Test]
+ public virtual void TestRefCounts1()
+ {
+ Directory dir1 = GetDir1(Random());
+ Directory dir2 = GetDir2(Random());
+ DirectoryReader ir1, ir2;
+ // close subreaders, ParallelReader will not change refCounts, but close on its own close
+ ParallelCompositeReader pr = new ParallelCompositeReader(ir1 = DirectoryReader.Open(dir1), ir2 = DirectoryReader.Open(dir2));
+ IndexReader psub1 = pr.GetSequentialSubReaders()[0];
+ // check RefCounts
+ Assert.AreEqual(1, ir1.RefCount);
+ Assert.AreEqual(1, ir2.RefCount);
+ Assert.AreEqual(1, psub1.RefCount);
+ pr.Dispose();
+ Assert.AreEqual(0, ir1.RefCount);
+ Assert.AreEqual(0, ir2.RefCount);
+ Assert.AreEqual(0, psub1.RefCount);
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void TestRefCounts2()
+ {
+ Directory dir1 = GetDir1(Random());
+ Directory dir2 = GetDir2(Random());
+ DirectoryReader ir1 = DirectoryReader.Open(dir1);
+ DirectoryReader ir2 = DirectoryReader.Open(dir2);
+
+ // don't close subreaders, so ParallelReader will increment refcounts
+ ParallelCompositeReader pr = new ParallelCompositeReader(false, ir1, ir2);
+ IndexReader psub1 = pr.GetSequentialSubReaders()[0];
+ // check RefCounts
+ Assert.AreEqual(2, ir1.RefCount);
+ Assert.AreEqual(2, ir2.RefCount);
+ Assert.AreEqual(1, psub1.RefCount, "refCount must be 1, as the synthetic reader was created by ParallelCompositeReader");
+ pr.Dispose();
+ Assert.AreEqual(1, ir1.RefCount);
+ Assert.AreEqual(1, ir2.RefCount);
+ Assert.AreEqual(0, psub1.RefCount, "refcount must be 0 because parent was closed");
+ ir1.Dispose();
+ ir2.Dispose();
+ Assert.AreEqual(0, ir1.RefCount);
+ Assert.AreEqual(0, ir2.RefCount);
+ Assert.AreEqual(0, psub1.RefCount, "refcount should not change anymore");
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ // closeSubreaders=false
+ [Test]
+ public virtual void TestReaderClosedListener1()
+ {
+ Directory dir1 = GetDir1(Random());
+ CompositeReader ir1 = DirectoryReader.Open(dir1);
+
+ // with overlapping
+ ParallelCompositeReader pr = new ParallelCompositeReader(false, new CompositeReader[] { ir1 }, new CompositeReader[] { ir1 });
+
+ int[] listenerClosedCount = new int[1];
+
+ Assert.AreEqual(3, pr.Leaves.Count);
+
+ foreach (AtomicReaderContext cxt in pr.Leaves)
+ {
+ cxt.Reader.AddReaderClosedListener(new ReaderClosedListenerAnonymousInnerClassHelper(this, listenerClosedCount));
+ }
+ pr.Dispose();
+ ir1.Dispose();
+ Assert.AreEqual(3, listenerClosedCount[0]);
+ dir1.Dispose();
+ }
+
+ private class ReaderClosedListenerAnonymousInnerClassHelper : IReaderClosedListener
+ {
+ private readonly TestParallelCompositeReader OuterInstance;
+
+ private int[] ListenerClosedCount;
+
+ public ReaderClosedListenerAnonymousInnerClassHelper(TestParallelCompositeReader outerInstance, int[] listenerClosedCount)
+ {
+ this.OuterInstance = outerInstance;
+ this.ListenerClosedCount = listenerClosedCount;
+ }
+
+ public void OnClose(IndexReader reader)
+ {
+ ListenerClosedCount[0]++;
+ }
+ }
+
+ // closeSubreaders=true
+ [Test]
+ public virtual void TestReaderClosedListener2()
+ {
+ Directory dir1 = GetDir1(Random());
+ CompositeReader ir1 = DirectoryReader.Open(dir1);
+
+ // with overlapping
+ ParallelCompositeReader pr = new ParallelCompositeReader(true, new CompositeReader[] { ir1 }, new CompositeReader[] { ir1 });
+
+ int[] listenerClosedCount = new int[1];
+
+ Assert.AreEqual(3, pr.Leaves.Count);
+
+ foreach (AtomicReaderContext cxt in pr.Leaves)
+ {
+ cxt.Reader.AddReaderClosedListener(new ReaderClosedListenerAnonymousInnerClassHelper2(this, listenerClosedCount));
+ }
+ pr.Dispose();
+ Assert.AreEqual(3, listenerClosedCount[0]);
+ dir1.Dispose();
+ }
+
+ private class ReaderClosedListenerAnonymousInnerClassHelper2 : IReaderClosedListener
+ {
+ private readonly TestParallelCompositeReader OuterInstance;
+
+ private int[] ListenerClosedCount;
+
+ public ReaderClosedListenerAnonymousInnerClassHelper2(TestParallelCompositeReader outerInstance, int[] listenerClosedCount)
+ {
+ this.OuterInstance = outerInstance;
+ this.ListenerClosedCount = listenerClosedCount;
+ }
+
+ public void OnClose(IndexReader reader)
+ {
+ ListenerClosedCount[0]++;
+ }
+ }
+
+ [Test]
+ public virtual void TestCloseInnerReader()
+ {
+ Directory dir1 = GetDir1(Random());
+ CompositeReader ir1 = DirectoryReader.Open(dir1);
+ Assert.AreEqual(1, ir1.GetSequentialSubReaders()[0].RefCount);
+
+ // with overlapping
+ ParallelCompositeReader pr = new ParallelCompositeReader(true, new CompositeReader[] { ir1 }, new CompositeReader[] { ir1 });
+
+ IndexReader psub = pr.GetSequentialSubReaders()[0];
+ Assert.AreEqual(1, psub.RefCount);
+
+ ir1.Dispose();
+
+ Assert.AreEqual(1, psub.RefCount, "refCount of synthetic subreader should be unchanged");
+ try
+ {
+ psub.Document(0);
+ Assert.Fail("Subreader should be already closed because inner reader was closed!");
+ }
+#pragma warning disable 168
+ catch (AlreadyClosedException e)
+#pragma warning restore 168
+ {
+ // pass
+ }
+
+ try
+ {
+ pr.Document(0);
+ Assert.Fail("ParallelCompositeReader should be already closed because inner reader was closed!");
+ }
+#pragma warning disable 168
+ catch (AlreadyClosedException e)
+#pragma warning restore 168
+ {
+ // pass
+ }
+
+ // noop:
+ pr.Dispose();
+ Assert.AreEqual(0, psub.RefCount);
+ dir1.Dispose();
+ }
+
+ [Test]
+ public virtual void TestIncompatibleIndexes1()
+ {
+ // two documents:
+ Directory dir1 = GetDir1(Random());
+
+ // one document only:
+ Directory dir2 = NewDirectory();
+ IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document d3 = new Document();
+
+ d3.Add(NewTextField("f3", "v1", Field.Store.YES));
+ w2.AddDocument(d3);
+ w2.Dispose();
+
+ DirectoryReader ir1 = DirectoryReader.Open(dir1), ir2 = DirectoryReader.Open(dir2);
+ try
+ {
+ new ParallelCompositeReader(ir1, ir2);
+ Assert.Fail("didn't get expected exception: indexes don't have same number of documents");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // expected exception
+ }
+ try
+ {
+ new ParallelCompositeReader(Random().NextBoolean(), ir1, ir2);
+ Assert.Fail("didn't get expected exception: indexes don't have same number of documents");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // expected exception
+ }
+ Assert.AreEqual(1, ir1.RefCount);
+ Assert.AreEqual(1, ir2.RefCount);
+ ir1.Dispose();
+ ir2.Dispose();
+ Assert.AreEqual(0, ir1.RefCount);
+ Assert.AreEqual(0, ir2.RefCount);
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void TestIncompatibleIndexes2()
+ {
+ Directory dir1 = GetDir1(Random());
+ Directory dir2 = GetInvalidStructuredDir2(Random());
+
+ DirectoryReader ir1 = DirectoryReader.Open(dir1), ir2 = DirectoryReader.Open(dir2);
+ CompositeReader[] readers = new CompositeReader[] { ir1, ir2 };
+ try
+ {
+ new ParallelCompositeReader(readers);
+ Assert.Fail("didn't get expected exception: indexes don't have same subreader structure");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // expected exception
+ }
+ try
+ {
+ new ParallelCompositeReader(Random().NextBoolean(), readers, readers);
+ Assert.Fail("didn't get expected exception: indexes don't have same subreader structure");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // expected exception
+ }
+ Assert.AreEqual(1, ir1.RefCount);
+ Assert.AreEqual(1, ir2.RefCount);
+ ir1.Dispose();
+ ir2.Dispose();
+ Assert.AreEqual(0, ir1.RefCount);
+ Assert.AreEqual(0, ir2.RefCount);
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void TestIncompatibleIndexes3()
+ {
+ Directory dir1 = GetDir1(Random());
+ Directory dir2 = GetDir2(Random());
+
+ CompositeReader ir1 = new MultiReader(DirectoryReader.Open(dir1), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(dir1))), ir2 = new MultiReader(DirectoryReader.Open(dir2), DirectoryReader.Open(dir2));
+ CompositeReader[] readers = new CompositeReader[] { ir1, ir2 };
+ try
+ {
+ new ParallelCompositeReader(readers);
+ Assert.Fail("didn't get expected exception: indexes don't have same subreader structure");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // expected exception
+ }
+ try
+ {
+ new ParallelCompositeReader(Random().NextBoolean(), readers, readers);
+ Assert.Fail("didn't get expected exception: indexes don't have same subreader structure");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException e)
+#pragma warning restore 168
+ {
+ // expected exception
+ }
+ Assert.AreEqual(1, ir1.RefCount);
+ Assert.AreEqual(1, ir2.RefCount);
+ ir1.Dispose();
+ ir2.Dispose();
+ Assert.AreEqual(0, ir1.RefCount);
+ Assert.AreEqual(0, ir2.RefCount);
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void TestIgnoreStoredFields()
+ {
+ Directory dir1 = GetDir1(Random());
+ Directory dir2 = GetDir2(Random());
+ CompositeReader ir1 = DirectoryReader.Open(dir1);
+ CompositeReader ir2 = DirectoryReader.Open(dir2);
+
+ // with overlapping
+ ParallelCompositeReader pr = new ParallelCompositeReader(false, new CompositeReader[] { ir1, ir2 }, new CompositeReader[] { ir1 });
+ Assert.AreEqual("v1", pr.Document(0).Get("f1"));
+ Assert.AreEqual("v1", pr.Document(0).Get("f2"));
+ Assert.IsNull(pr.Document(0).Get("f3"));
+ Assert.IsNull(pr.Document(0).Get("f4"));
+ // check that fields are there
+ AtomicReader slow = SlowCompositeReaderWrapper.Wrap(pr);
+ Assert.IsNotNull(slow.Terms("f1"));
+ Assert.IsNotNull(slow.Terms("f2"));
+ Assert.IsNotNull(slow.Terms("f3"));
+ Assert.IsNotNull(slow.Terms("f4"));
+ pr.Dispose();
+
+ // no stored fields at all
+ pr = new ParallelCompositeReader(false, new CompositeReader[] { ir2 }, new CompositeReader[0]);
+ Assert.IsNull(pr.Document(0).Get("f1"));
+ Assert.IsNull(pr.Document(0).Get("f2"));
+ Assert.IsNull(pr.Document(0).Get("f3"));
+ Assert.IsNull(pr.Document(0).Get("f4"));
+ // check that fields are there
+ slow = SlowCompositeReaderWrapper.Wrap(pr);
+ Assert.IsNull(slow.Terms("f1"));
+ Assert.IsNull(slow.Terms("f2"));
+ Assert.IsNotNull(slow.Terms("f3"));
+ Assert.IsNotNull(slow.Terms("f4"));
+ pr.Dispose();
+
+ // without overlapping
+ pr = new ParallelCompositeReader(true, new CompositeReader[] { ir2 }, new CompositeReader[] { ir1 });
+ Assert.AreEqual("v1", pr.Document(0).Get("f1"));
+ Assert.AreEqual("v1", pr.Document(0).Get("f2"));
+ Assert.IsNull(pr.Document(0).Get("f3"));
+ Assert.IsNull(pr.Document(0).Get("f4"));
+ // check that fields are there
+ slow = SlowCompositeReaderWrapper.Wrap(pr);
+ Assert.IsNull(slow.Terms("f1"));
+ Assert.IsNull(slow.Terms("f2"));
+ Assert.IsNotNull(slow.Terms("f3"));
+ Assert.IsNotNull(slow.Terms("f4"));
+ pr.Dispose();
+
+ // no main readers
+ try
+ {
+ new ParallelCompositeReader(true, new CompositeReader[0], new CompositeReader[] { ir1 });
+ Assert.Fail("didn't get expected exception: need a non-empty main-reader array");
+ }
+#pragma warning disable 168
+ catch (System.ArgumentException iae)
+#pragma warning restore 168
+ {
+ // pass
+ }
+
+ dir1.Dispose();
+ dir2.Dispose();
+ }
+
+ [Test]
+ public virtual void TestToString()
+ {
+ Directory dir1 = GetDir1(Random());
+ CompositeReader ir1 = DirectoryReader.Open(dir1);
+ ParallelCompositeReader pr = new ParallelCompositeReader(new CompositeReader[] { ir1 });
+
+ string s = pr.ToString();
+ Assert.IsTrue(s.StartsWith("ParallelCompositeReader(ParallelAtomicReader("), "toString incorrect: " + s);
+
+ pr.Dispose();
+ dir1.Dispose();
+ }
+
+ [Test]
+ public virtual void TestToStringCompositeComposite()
+ {
+ Directory dir1 = GetDir1(Random());
+ CompositeReader ir1 = DirectoryReader.Open(dir1);
+ ParallelCompositeReader pr = new ParallelCompositeReader(new CompositeReader[] { new MultiReader(ir1) });
+
+ string s = pr.ToString();
+
+ Assert.IsTrue(s.StartsWith("ParallelCompositeReader(ParallelCompositeReaderAnonymousInnerClassHelper(ParallelAtomicReader("), "toString incorrect: " + s);
+
+ pr.Dispose();
+ dir1.Dispose();
+ }
+
+ private void QueryTest(Query query)
+ {
+ ScoreDoc[] parallelHits = Parallel_Renamed.Search(query, null, 1000).ScoreDocs;
+ ScoreDoc[] singleHits = Single_Renamed.Search(query, null, 1000).ScoreDocs;
+ Assert.AreEqual(parallelHits.Length, singleHits.Length);
+ for (int i = 0; i < parallelHits.Length; i++)
+ {
+ Assert.AreEqual(parallelHits[i].Score, singleHits[i].Score, 0.001f);
+ Document docParallel = Parallel_Renamed.Doc(parallelHits[i].Doc);
+ Document docSingle = Single_Renamed.Doc(singleHits[i].Doc);
+ Assert.AreEqual(docParallel.Get("f1"), docSingle.Get("f1"));
+ Assert.AreEqual(docParallel.Get("f2"), docSingle.Get("f2"));
+ Assert.AreEqual(docParallel.Get("f3"), docSingle.Get("f3"));
+ Assert.AreEqual(docParallel.Get("f4"), docSingle.Get("f4"));
+ }
+ }
+
+ // Fields 1-4 indexed together:
+ private IndexSearcher Single(Random random, bool compositeComposite)
+ {
+ Dir = NewDirectory();
+ IndexWriter w = new IndexWriter(Dir, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ Document d1 = new Document();
+ d1.Add(NewTextField("f1", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f2", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f3", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f4", "v1", Field.Store.YES));
+ w.AddDocument(d1);
+ Document d2 = new Document();
+ d2.Add(NewTextField("f1", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f2", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f3", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f4", "v2", Field.Store.YES));
+ w.AddDocument(d2);
+ Document d3 = new Document();
+ d3.Add(NewTextField("f1", "v3", Field.Store.YES));
+ d3.Add(NewTextField("f2", "v3", Field.Store.YES));
+ d3.Add(NewTextField("f3", "v3", Field.Store.YES));
+ d3.Add(NewTextField("f4", "v3", Field.Store.YES));
+ w.AddDocument(d3);
+ Document d4 = new Document();
+ d4.Add(NewTextField("f1", "v4", Field.Store.YES));
+ d4.Add(NewTextField("f2", "v4", Field.Store.YES));
+ d4.Add(NewTextField("f3", "v4", Field.Store.YES));
+ d4.Add(NewTextField("f4", "v4", Field.Store.YES));
+ w.AddDocument(d4);
+ w.Dispose();
+
+ CompositeReader ir;
+ if (compositeComposite)
+ {
+ ir = new MultiReader(DirectoryReader.Open(Dir), DirectoryReader.Open(Dir));
+ }
+ else
+ {
+ ir = DirectoryReader.Open(Dir);
+ }
+ return NewSearcher(ir);
+ }
+
+ // Fields 1 & 2 in one index, 3 & 4 in other, with ParallelReader:
+ private IndexSearcher Parallel(Random random, bool compositeComposite)
+ {
+ Dir1 = GetDir1(random);
+ Dir2 = GetDir2(random);
+ CompositeReader rd1, rd2;
+ if (compositeComposite)
+ {
+ rd1 = new MultiReader(DirectoryReader.Open(Dir1), DirectoryReader.Open(Dir1));
+ rd2 = new MultiReader(DirectoryReader.Open(Dir2), DirectoryReader.Open(Dir2));
+ Assert.AreEqual(2, rd1.Context.Children.Count);
+ Assert.AreEqual(2, rd2.Context.Children.Count);
+ }
+ else
+ {
+ rd1 = DirectoryReader.Open(Dir1);
+ rd2 = DirectoryReader.Open(Dir2);
+ Assert.AreEqual(3, rd1.Context.Children.Count);
+ Assert.AreEqual(3, rd2.Context.Children.Count);
+ }
+ ParallelCompositeReader pr = new ParallelCompositeReader(rd1, rd2);
+ return NewSearcher(pr);
+ }
+
+ // subreader structure: (1,2,1)
+ private Directory GetDir1(Random random)
+ {
+ Directory dir1 = NewDirectory();
+ IndexWriter w1 = new IndexWriter(dir1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES));
+ Document d1 = new Document();
+ d1.Add(NewTextField("f1", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f2", "v1", Field.Store.YES));
+ w1.AddDocument(d1);
+ w1.Commit();
+ Document d2 = new Document();
+ d2.Add(NewTextField("f1", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f2", "v2", Field.Store.YES));
+ w1.AddDocument(d2);
+ Document d3 = new Document();
+ d3.Add(NewTextField("f1", "v3", Field.Store.YES));
+ d3.Add(NewTextField("f2", "v3", Field.Store.YES));
+ w1.AddDocument(d3);
+ w1.Commit();
+ Document d4 = new Document();
+ d4.Add(NewTextField("f1", "v4", Field.Store.YES));
+ d4.Add(NewTextField("f2", "v4", Field.Store.YES));
+ w1.AddDocument(d4);
+ w1.Dispose();
+ return dir1;
+ }
+
+ // subreader structure: (1,2,1)
+ private Directory GetDir2(Random random)
+ {
+ Directory dir2 = NewDirectory();
+ IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES));
+ Document d1 = new Document();
+ d1.Add(NewTextField("f3", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f4", "v1", Field.Store.YES));
+ w2.AddDocument(d1);
+ w2.Commit();
+ Document d2 = new Document();
+ d2.Add(NewTextField("f3", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f4", "v2", Field.Store.YES));
+ w2.AddDocument(d2);
+ Document d3 = new Document();
+ d3.Add(NewTextField("f3", "v3", Field.Store.YES));
+ d3.Add(NewTextField("f4", "v3", Field.Store.YES));
+ w2.AddDocument(d3);
+ w2.Commit();
+ Document d4 = new Document();
+ d4.Add(NewTextField("f3", "v4", Field.Store.YES));
+ d4.Add(NewTextField("f4", "v4", Field.Store.YES));
+ w2.AddDocument(d4);
+ w2.Dispose();
+ return dir2;
+ }
+
+ // this dir has a different subreader structure (1,1,2);
+ private Directory GetInvalidStructuredDir2(Random random)
+ {
+ Directory dir2 = NewDirectory();
+ IndexWriter w2 = new IndexWriter(dir2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).SetMergePolicy(NoMergePolicy.NO_COMPOUND_FILES));
+ Document d1 = new Document();
+ d1.Add(NewTextField("f3", "v1", Field.Store.YES));
+ d1.Add(NewTextField("f4", "v1", Field.Store.YES));
+ w2.AddDocument(d1);
+ w2.Commit();
+ Document d2 = new Document();
+ d2.Add(NewTextField("f3", "v2", Field.Store.YES));
+ d2.Add(NewTextField("f4", "v2", Field.Store.YES));
+ w2.AddDocument(d2);
+ w2.Commit();
+ Document d3 = new Document();
+ d3.Add(NewTextField("f3", "v3", Field.Store.YES));
+ d3.Add(NewTextField("f4", "v3", Field.Store.YES));
+ w2.AddDocument(d3);
+ Document d4 = new Document();
+ d4.Add(NewTextField("f3", "v4", Field.Store.YES));
+ d4.Add(NewTextField("f4", "v4", Field.Store.YES));
+ w2.AddDocument(d4);
+ w2.Dispose();
+ return dir2;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestParallelReaderEmptyIndex.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestParallelReaderEmptyIndex.cs b/src/Lucene.Net.Tests/Index/TestParallelReaderEmptyIndex.cs
new file mode 100644
index 0000000..f51128e
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestParallelReaderEmptyIndex.cs
@@ -0,0 +1,162 @@
+using System;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using NUnit.Framework;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Documents.Document;
+ using Field = Field;
+ using FieldType = FieldType;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using TextField = TextField;
+
+ /// <summary>
+ /// Some tests for <seealso cref="ParallelAtomicReader"/>s with empty indexes
+ /// </summary>
+ [TestFixture]
+ public class TestParallelReaderEmptyIndex : LuceneTestCase
+ {
+ /// <summary>
+ /// Creates two empty indexes and wraps a ParallelReader around. Adding this
+ /// reader to a new index should not throw any exception.
+ /// </summary>
+ [Test]
+ public virtual void TestEmptyIndex()
+ {
+ Directory rd1 = NewDirectory();
+ IndexWriter iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ iw.Dispose();
+ // create a copy:
+ Directory rd2 = NewDirectory(rd1);
+
+ Directory rdOut = NewDirectory();
+
+ IndexWriter iwOut = new IndexWriter(rdOut, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ ParallelAtomicReader apr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(rd1)), SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(rd2)));
+
+ // When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum)
+ iwOut.AddIndexes(apr);
+ iwOut.ForceMerge(1);
+
+ // 2nd try with a readerless parallel reader
+ iwOut.AddIndexes(new ParallelAtomicReader());
+ iwOut.ForceMerge(1);
+
+ ParallelCompositeReader cpr = new ParallelCompositeReader(DirectoryReader.Open(rd1), DirectoryReader.Open(rd2));
+
+ // When unpatched, Lucene crashes here with a NoSuchElementException (caused by ParallelTermEnum)
+ iwOut.AddIndexes(cpr);
+ iwOut.ForceMerge(1);
+
+ // 2nd try with a readerless parallel reader
+ iwOut.AddIndexes(new ParallelCompositeReader());
+ iwOut.ForceMerge(1);
+
+ iwOut.Dispose();
+ rdOut.Dispose();
+ rd1.Dispose();
+ rd2.Dispose();
+ }
+
+ /// <summary>
+ /// this method creates an empty index (numFields=0, numDocs=0) but is marked
+ /// to have TermVectors. Adding this index to another index should not throw
+ /// any exception.
+ /// </summary>
+ [Test]
+ public virtual void TestEmptyIndexWithVectors()
+ {
+ Directory rd1 = NewDirectory();
+ {
+ if (VERBOSE)
+ {
+ Console.WriteLine("\nTEST: make 1st writer");
+ }
+ IndexWriter iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ Field idField = NewTextField("id", "", Field.Store.NO);
+ doc.Add(idField);
+ FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
+ customType.StoreTermVectors = true;
+ doc.Add(NewField("test", "", customType));
+ idField.SetStringValue("1");
+ iw.AddDocument(doc);
+ doc.Add(NewTextField("test", "", Field.Store.NO));
+ idField.SetStringValue("2");
+ iw.AddDocument(doc);
+ iw.Dispose();
+
+ IndexWriterConfig dontMergeConfig = (new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random()))).SetMergePolicy(NoMergePolicy.COMPOUND_FILES);
+ if (VERBOSE)
+ {
+ Console.WriteLine("\nTEST: make 2nd writer");
+ }
+ IndexWriter writer = new IndexWriter(rd1, dontMergeConfig);
+
+ writer.DeleteDocuments(new Term("id", "1"));
+ writer.Dispose();
+ IndexReader ir = DirectoryReader.Open(rd1);
+ Assert.AreEqual(2, ir.MaxDoc);
+ Assert.AreEqual(1, ir.NumDocs);
+ ir.Dispose();
+
+ iw = new IndexWriter(rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())).SetOpenMode(OpenMode.APPEND));
+ iw.ForceMerge(1);
+ iw.Dispose();
+ }
+
+ Directory rd2 = NewDirectory();
+ {
+ IndexWriter iw = new IndexWriter(rd2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ Document doc = new Document();
+ iw.AddDocument(doc);
+ iw.Dispose();
+ }
+
+ Directory rdOut = NewDirectory();
+
+ IndexWriter iwOut = new IndexWriter(rdOut, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+ DirectoryReader reader1, reader2;
+ ParallelAtomicReader pr = new ParallelAtomicReader(SlowCompositeReaderWrapper.Wrap(reader1 = DirectoryReader.Open(rd1)), SlowCompositeReaderWrapper.Wrap(reader2 = DirectoryReader.Open(rd2)));
+
+ // When unpatched, Lucene crashes here with an ArrayIndexOutOfBoundsException (caused by TermVectorsWriter)
+ iwOut.AddIndexes(pr);
+
+ // ParallelReader closes any IndexReader you added to it:
+ pr.Dispose();
+
+ // assert subreaders were closed
+ Assert.AreEqual(0, reader1.RefCount);
+ Assert.AreEqual(0, reader2.RefCount);
+
+ rd1.Dispose();
+ rd2.Dispose();
+
+ iwOut.ForceMerge(1);
+ iwOut.Dispose();
+
+ rdOut.Dispose();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/96822396/src/Lucene.Net.Tests/Index/TestParallelTermEnum.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests/Index/TestParallelTermEnum.cs b/src/Lucene.Net.Tests/Index/TestParallelTermEnum.cs
new file mode 100644
index 0000000..9b6ac85
--- /dev/null
+++ b/src/Lucene.Net.Tests/Index/TestParallelTermEnum.cs
@@ -0,0 +1,127 @@
+using System.Collections.Generic;
+using Lucene.Net.Documents;
+
+namespace Lucene.Net.Index
+{
+ using NUnit.Framework;
+ using IBits = Lucene.Net.Util.IBits;
+ using BytesRef = Lucene.Net.Util.BytesRef;
+ using Directory = Lucene.Net.Store.Directory;
+ using DocIdSetIterator = Lucene.Net.Search.DocIdSetIterator;
+ using Document = Documents.Document;
+ using Field = Field;
+ using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ using MockAnalyzer = Lucene.Net.Analysis.MockAnalyzer;
+ using TestUtil = Lucene.Net.Util.TestUtil;
+
+ [TestFixture]
+ public class TestParallelTermEnum : LuceneTestCase
+ {
+ private AtomicReader Ir1;
+ private AtomicReader Ir2;
+ private Directory Rd1;
+ private Directory Rd2;
+
+ [SetUp]
+ public override void SetUp()
+ {
+ base.SetUp();
+ Document doc;
+ Rd1 = NewDirectory();
+ IndexWriter iw1 = new IndexWriter(Rd1, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ doc = new Document();
+ doc.Add(NewTextField("field1", "the quick brown fox jumps", Field.Store.YES));
+ doc.Add(NewTextField("field2", "the quick brown fox jumps", Field.Store.YES));
+ iw1.AddDocument(doc);
+
+ iw1.Dispose();
+ Rd2 = NewDirectory();
+ IndexWriter iw2 = new IndexWriter(Rd2, NewIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(Random())));
+
+ doc = new Document();
+ doc.Add(NewTextField("field1", "the fox jumps over the lazy dog", Field.Store.YES));
+ doc.Add(NewTextField("field3", "the fox jumps over the lazy dog", Field.Store.YES));
+ iw2.AddDocument(doc);
+
+ iw2.Dispose();
+
+ this.Ir1 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(Rd1));
+ this.Ir2 = SlowCompositeReaderWrapper.Wrap(DirectoryReader.Open(Rd2));
+ }
+
+ [TearDown]
+ public override void TearDown()
+ {
+ Ir1.Dispose();
+ Ir2.Dispose();
+ Rd1.Dispose();
+ Rd2.Dispose();
+ base.TearDown();
+ }
+
+ private void CheckTerms(Terms terms, IBits liveDocs, params string[] termsList)
+ {
+ Assert.IsNotNull(terms);
+ TermsEnum te = terms.GetIterator(null);
+
+ foreach (string t in termsList)
+ {
+ BytesRef b = te.Next();
+ Assert.IsNotNull(b);
+ Assert.AreEqual(t, b.Utf8ToString());
+ DocsEnum td = TestUtil.Docs(Random(), te, liveDocs, null, DocsEnum.FLAG_NONE);
+ Assert.IsTrue(td.NextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ Assert.AreEqual(0, td.DocID);
+ Assert.AreEqual(td.NextDoc(), DocIdSetIterator.NO_MORE_DOCS);
+ }
+ Assert.IsNull(te.Next());
+ }
+
+ [Test]
+ public virtual void Test1()
+ {
+ ParallelAtomicReader pr = new ParallelAtomicReader(Ir1, Ir2);
+
+ IBits liveDocs = pr.LiveDocs;
+
+ Fields fields = pr.Fields;
+ IEnumerator<string> fe = fields.GetEnumerator();
+
+ fe.MoveNext();
+ string f = fe.Current;
+ Assert.AreEqual("field1", f);
+ CheckTerms(fields.GetTerms(f), liveDocs, "brown", "fox", "jumps", "quick", "the");
+
+ fe.MoveNext();
+ f = fe.Current;
+ Assert.AreEqual("field2", f);
+ CheckTerms(fields.GetTerms(f), liveDocs, "brown", "fox", "jumps", "quick", "the");
+
+ fe.MoveNext();
+ f = fe.Current;
+ Assert.AreEqual("field3", f);
+ CheckTerms(fields.GetTerms(f), liveDocs, "dog", "fox", "jumps", "lazy", "over", "the");
+
+ Assert.IsFalse(fe.MoveNext());
+ }
+ }
+}
\ No newline at end of file