You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ar...@apache.org on 2008/07/15 23:44:10 UTC
svn commit: r677059 [10/19] - in /incubator/lucene.net/trunk/C#/src: ./
Demo/DeleteFiles/ Demo/DemoLib/ Demo/IndexFiles/ Demo/IndexHtml/
Demo/SearchFiles/ Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Index/
Lucene.Net/Search/ Lucene.Net/Search/Function/...
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestParallelReader.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs Tue Jul 15 14:44:04 2008
@@ -19,10 +19,13 @@
using NUnit.Framework;
-using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
using Document = Lucene.Net.Documents.Document;
using Field = Lucene.Net.Documents.Field;
using MapFieldSelector = Lucene.Net.Documents.MapFieldSelector;
+using Directory = Lucene.Net.Store.Directory;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
+using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
using BooleanQuery = Lucene.Net.Search.BooleanQuery;
using Hits = Lucene.Net.Search.Hits;
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
@@ -30,27 +33,27 @@
using Searcher = Lucene.Net.Search.Searcher;
using TermQuery = Lucene.Net.Search.TermQuery;
using Occur = Lucene.Net.Search.BooleanClause.Occur;
-using Directory = Lucene.Net.Store.Directory;
-using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Index
{
[TestFixture]
- public class TestParallelReader
+ public class TestParallelReader : LuceneTestCase
{
private Searcher parallel;
private Searcher single;
[SetUp]
- public virtual void SetUp()
+ public override void SetUp()
{
+ base.SetUp();
single = Single();
parallel = Parallel();
}
[Test]
- public virtual void TestQueries()
+ public virtual void TestQueries()
{
QueryTest(new TermQuery(new Term("f1", "v1")));
QueryTest(new TermQuery(new Term("f1", "v2")));
@@ -68,7 +71,7 @@
}
[Test]
- public virtual void TestFieldNames()
+ public virtual void TestFieldNames()
{
Directory dir1 = GetDir1();
Directory dir2 = GetDir2();
@@ -83,37 +86,37 @@
Assert.IsTrue(CollectionContains(fieldNames, "f4"));
}
- [Test]
- public virtual void TestDocument()
- {
- Directory dir1 = GetDir1();
- Directory dir2 = GetDir2();
- ParallelReader pr = new ParallelReader();
- pr.Add(IndexReader.Open(dir1));
- pr.Add(IndexReader.Open(dir2));
-
- Lucene.Net.Documents.Document doc11 = pr.Document(0, new MapFieldSelector(new System.String[]{"f1"}));
- Lucene.Net.Documents.Document doc24 = pr.Document(1, new MapFieldSelector(new System.Collections.ArrayList(new System.String[]{"f4"})));
- Lucene.Net.Documents.Document doc223 = pr.Document(1, new MapFieldSelector(new System.String[]{"f2", "f3"}));
-
- Assert.AreEqual(1, doc11.GetFields().Count);
- Assert.AreEqual(1, doc24.GetFields().Count);
- Assert.AreEqual(2, doc223.GetFields().Count);
-
- Assert.AreEqual("v1", doc11.Get("f1"));
- Assert.AreEqual("v2", doc24.Get("f4"));
- Assert.AreEqual("v2", doc223.Get("f2"));
- Assert.AreEqual("v2", doc223.Get("f3"));
- }
+ [Test]
+ public virtual void TestDocument()
+ {
+ Directory dir1 = GetDir1();
+ Directory dir2 = GetDir2();
+ ParallelReader pr = new ParallelReader();
+ pr.Add(IndexReader.Open(dir1));
+ pr.Add(IndexReader.Open(dir2));
+
+ Lucene.Net.Documents.Document doc11 = pr.Document(0, new MapFieldSelector(new System.String[]{"f1"}));
+ Lucene.Net.Documents.Document doc24 = pr.Document(1, new MapFieldSelector(new System.Collections.ArrayList(new System.String[]{"f4"})));
+ Lucene.Net.Documents.Document doc223 = pr.Document(1, new MapFieldSelector(new System.String[]{"f2", "f3"}));
+
+ Assert.AreEqual(1, doc11.GetFields().Count);
+ Assert.AreEqual(1, doc24.GetFields().Count);
+ Assert.AreEqual(2, doc223.GetFields().Count);
+
+ Assert.AreEqual("v1", doc11.Get("f1"));
+ Assert.AreEqual("v2", doc24.Get("f4"));
+ Assert.AreEqual("v2", doc223.Get("f2"));
+ Assert.AreEqual("v2", doc223.Get("f3"));
+ }
- [Test]
- public virtual void TestIncompatibleIndexes()
+ [Test]
+ public virtual void TestIncompatibleIndexes()
{
// two documents:
Directory dir1 = GetDir1();
// one document only:
- Directory dir2 = new RAMDirectory();
+ Directory dir2 = new MockRAMDirectory();
IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true);
Lucene.Net.Documents.Document d3 = new Lucene.Net.Documents.Document();
d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.TOKENIZED));
@@ -127,12 +130,89 @@
pr.Add(IndexReader.Open(dir2));
Assert.Fail("didn't get exptected exception: indexes don't have same number of documents");
}
- catch (System.ArgumentException e)
+ catch (System.ArgumentException)
{
// expected exception
}
}
+ [Test]
+ public virtual void TestIsCurrent()
+ {
+ Directory dir1 = GetDir1();
+ Directory dir2 = GetDir1();
+ ParallelReader pr = new ParallelReader();
+ pr.Add(IndexReader.Open(dir1));
+ pr.Add(IndexReader.Open(dir2));
+
+ Assert.IsTrue(pr.IsCurrent());
+ IndexReader modifier = IndexReader.Open(dir1);
+ modifier.SetNorm(0, "f1", 100);
+ modifier.Close();
+
+ // one of the two IndexReaders which ParallelReader is using
+ // is not current anymore
+ Assert.IsFalse(pr.IsCurrent());
+
+ modifier = IndexReader.Open(dir2);
+ modifier.SetNorm(0, "f3", 100);
+ modifier.Close();
+
+ // now both are not current anymore
+ Assert.IsFalse(pr.IsCurrent());
+ }
+
+ [Test]
+ public virtual void TestIsOptimized()
+ {
+ Directory dir1 = GetDir1();
+ Directory dir2 = GetDir1();
+
+ // add another document to ensure that the indexes are not optimized
+ IndexWriter modifier = new IndexWriter(dir1, new StandardAnalyzer());
+ Document d = new Document();
+ d.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+ modifier.AddDocument(d);
+ modifier.Close();
+
+ modifier = new IndexWriter(dir2, new StandardAnalyzer());
+ d = new Document();
+ d.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.TOKENIZED));
+ modifier.AddDocument(d);
+ modifier.Close();
+
+
+ ParallelReader pr = new ParallelReader();
+ pr.Add(IndexReader.Open(dir1));
+ pr.Add(IndexReader.Open(dir2));
+ Assert.IsFalse(pr.IsOptimized());
+ pr.Close();
+
+ modifier = new IndexWriter(dir1, new StandardAnalyzer());
+ modifier.Optimize();
+ modifier.Close();
+
+ pr = new ParallelReader();
+ pr.Add(IndexReader.Open(dir1));
+ pr.Add(IndexReader.Open(dir2));
+ // just one of the two indexes are optimized
+ Assert.IsFalse(pr.IsOptimized());
+ pr.Close();
+
+
+ modifier = new IndexWriter(dir2, new StandardAnalyzer());
+ modifier.Optimize();
+ modifier.Close();
+
+ pr = new ParallelReader();
+ pr.Add(IndexReader.Open(dir1));
+ pr.Add(IndexReader.Open(dir2));
+ // now both indexes are optimized
+ Assert.IsTrue(pr.IsOptimized());
+ pr.Close();
+ }
+
+
private void QueryTest(Query query)
{
Hits parallelHits = parallel.Search(query);
@@ -150,10 +230,10 @@
}
}
- // Fiels 1-4 indexed together:
+ // Fields 1-4 indexed together:
private Searcher Single()
{
- Directory dir = new RAMDirectory();
+ Directory dir = new MockRAMDirectory();
IndexWriter w = new IndexWriter(dir, new StandardAnalyzer(), true);
Lucene.Net.Documents.Document d1 = new Lucene.Net.Documents.Document();
d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.TOKENIZED));
@@ -185,7 +265,7 @@
private Directory GetDir1()
{
- Directory dir1 = new RAMDirectory();
+ Directory dir1 = new MockRAMDirectory();
IndexWriter w1 = new IndexWriter(dir1, new StandardAnalyzer(), true);
Lucene.Net.Documents.Document d1 = new Lucene.Net.Documents.Document();
d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.TOKENIZED));
@@ -215,16 +295,16 @@
return dir2;
}
- public static bool CollectionContains(System.Collections.ICollection col, System.String val)
- {
- for (System.Collections.IEnumerator iterator = col.GetEnumerator(); iterator.MoveNext(); )
- {
- System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iterator.Current;
- System.String s = fi.Key.ToString();
- if (s == val)
- return true;
- }
- return false;
- }
- }
+ public static bool CollectionContains(System.Collections.ICollection col, System.String val)
+ {
+ for (System.Collections.IEnumerator iterator = col.GetEnumerator(); iterator.MoveNext(); )
+ {
+ System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iterator.Current;
+ System.String s = fi.Key.ToString();
+ if (s == val)
+ return true;
+ }
+ return false;
+ }
+ }
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestParallelTermEnum.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs Tue Jul 15 14:44:04 2008
@@ -19,25 +19,28 @@
using NUnit.Framework;
-using SimpleAnalyzer = Lucene.Net.Analysis.SimpleAnalyzer;
using Document = Lucene.Net.Documents.Document;
using Field = Lucene.Net.Documents.Field;
using Index = Lucene.Net.Documents.Field.Index;
using Store = Lucene.Net.Documents.Field.Store;
using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using SimpleAnalyzer = Lucene.Net.Analysis.SimpleAnalyzer;
namespace Lucene.Net.Index
{
- [TestFixture]
- public class TestParallelTermEnum
+ [TestFixture]
+ public class TestParallelTermEnum : LuceneTestCase
{
private IndexReader ir1;
private IndexReader ir2;
- [SetUp]
- public virtual void SetUp()
+ [SetUp]
+ public override void SetUp()
{
+ base.SetUp();
+ base.SetUp();
Lucene.Net.Documents.Document doc;
RAMDirectory rd1 = new RAMDirectory();
@@ -65,14 +68,17 @@
this.ir2 = IndexReader.Open(rd2);
}
- [TearDown]
- public virtual void TearDown()
+ [TearDown]
+ public override void TearDown()
{
+ base.TearDown();
+ base.TearDown();
+
ir1.Close();
ir2.Close();
}
- [Test]
+ [Test]
public virtual void Test1()
{
ParallelReader pr = new ParallelReader();
Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestPayloads.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,718 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using NUnit.Framework;
+
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using Directory = Lucene.Net.Store.Directory;
+using FSDirectory = Lucene.Net.Store.FSDirectory;
+using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using Analyzer = Lucene.Net.Analysis.Analyzer;
+using Token = Lucene.Net.Analysis.Token;
+using TokenFilter = Lucene.Net.Analysis.TokenFilter;
+using TokenStream = Lucene.Net.Analysis.TokenStream;
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+using WhitespaceTokenizer = Lucene.Net.Analysis.WhitespaceTokenizer;
+
+namespace Lucene.Net.Index
+{
+
+
+ [TestFixture]
+ public class TestPayloads : LuceneTestCase
+ {
+ private class AnonymousClassThread : SupportClass.ThreadClass
+ {
+ public AnonymousClassThread(int numDocs, System.String field, Lucene.Net.Index.TestPayloads.ByteArrayPool pool, Lucene.Net.Index.IndexWriter writer, TestPayloads enclosingInstance)
+ {
+ InitBlock(numDocs, field, pool, writer, enclosingInstance);
+ }
+ private void InitBlock(int numDocs, System.String field, Lucene.Net.Index.TestPayloads.ByteArrayPool pool, Lucene.Net.Index.IndexWriter writer, TestPayloads enclosingInstance)
+ {
+ this.numDocs = numDocs;
+ this.field = field;
+ this.pool = pool;
+ this.writer = writer;
+ this.enclosingInstance = enclosingInstance;
+ }
+ private int numDocs;
+ private System.String field;
+ private Lucene.Net.Index.TestPayloads.ByteArrayPool pool;
+ private Lucene.Net.Index.IndexWriter writer;
+ private TestPayloads enclosingInstance;
+ public TestPayloads Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+ override public void Run()
+ {
+ try
+ {
+ for (int j = 0; j < numDocs; j++)
+ {
+ Document d = new Document();
+ d.Add(new Field(field, new PoolingPayloadTokenStream(pool)));
+ writer.AddDocument(d);
+ }
+ }
+ catch (System.Exception e)
+ {
+ System.Console.Error.WriteLine(e.StackTrace);
+ Assert.Fail(e.ToString());
+ }
+ }
+ }
+
+ // Simple tests to test the Payload class
+ [Test]
+ public virtual void TestPayload()
+ {
+ byte[] testData = System.Text.Encoding.UTF8.GetBytes("This is a test!");
+ Payload payload = new Payload(testData);
+ Assert.AreEqual(testData.Length, payload.Length(), "Wrong payload length.");
+
+ // test copyTo()
+ byte[] target = new byte[testData.Length - 1];
+ try
+ {
+ payload.CopyTo(target, 0);
+ Assert.Fail("Expected exception not thrown");
+ }
+ catch (System.Exception)
+ {
+ // expected exception
+ }
+
+ target = new byte[testData.Length + 3];
+ payload.CopyTo(target, 3);
+
+ for (int i = 0; i < testData.Length; i++)
+ {
+ Assert.AreEqual(testData[i], target[i + 3]);
+ }
+
+
+ // test toByteArray()
+ target = payload.ToByteArray();
+ AssertByteArrayEquals(testData, target);
+
+ // test byteAt()
+ for (int i = 0; i < testData.Length; i++)
+ {
+ Assert.AreEqual(payload.ByteAt(i), testData[i]);
+ }
+
+ try
+ {
+ payload.ByteAt(testData.Length + 1);
+ Assert.Fail("Expected exception not thrown");
+ }
+ catch (System.Exception)
+ {
+ // expected exception
+ }
+
+ Payload clone = (Payload) payload.Clone();
+ Assert.AreEqual(payload.Length(), clone.Length());
+ for (int i = 0; i < payload.Length(); i++)
+ {
+ Assert.AreEqual(payload.ByteAt(i), clone.ByteAt(i));
+ }
+ }
+
+ // Tests whether the DocumentWriter and SegmentMerger correctly enable the
+ // payload bit in the FieldInfo
+ [Test]
+ public virtual void TestPayloadFieldBit()
+ {
+ Directory ram = new RAMDirectory();
+ PayloadAnalyzer analyzer = new PayloadAnalyzer();
+ IndexWriter writer = new IndexWriter(ram, analyzer, true);
+ Document d = new Document();
+ // this field won't have any payloads
+ d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.TOKENIZED));
+ // this field will have payloads in all docs, however not for all term positions,
+ // so this field is used to check if the DocumentWriter correctly enables the payloads bit
+ // even if only some term positions have payloads
+ d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
+ // this field is used to verify if the SegmentMerger enables payloads for a field if it has payloads
+ // enabled in only some documents
+ d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.TOKENIZED));
+ // only add payload data for field f2
+ analyzer.SetPayloadData("f2", 1, System.Text.Encoding.UTF8.GetBytes("somedata"), 0, 1);
+ writer.AddDocument(d);
+ // flush
+ writer.Close();
+
+ // only one segment in the index, so we can cast to SegmentReader
+ SegmentReader reader = (SegmentReader) IndexReader.Open(ram);
+ FieldInfos fi = reader.FieldInfos();
+ Assert.IsFalse(fi.FieldInfo("f1").StorePayloads_ForNUnitTest, "Payload field bit should not be set.");
+ Assert.IsTrue(fi.FieldInfo("f2").StorePayloads_ForNUnitTest, "Payload field bit should be set.");
+ Assert.IsFalse(fi.FieldInfo("f3").StorePayloads_ForNUnitTest, "Payload field bit should not be set.");
+ reader.Close();
+
+ // now we add another document which has payloads for field f3 and verify if the SegmentMerger
+ // enabled payloads for that field
+ writer = new IndexWriter(ram, analyzer, true);
+ d = new Document();
+ d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.TOKENIZED));
+ // add payload data for field f2 and f3
+ analyzer.SetPayloadData("f2", System.Text.Encoding.UTF8.GetBytes("somedata"), 0, 1);
+ analyzer.SetPayloadData("f3", System.Text.Encoding.UTF8.GetBytes("somedata"), 0, 3);
+ writer.AddDocument(d);
+ // force merge
+ writer.Optimize();
+ // flush
+ writer.Close();
+
+ // only one segment in the index, so we can cast to SegmentReader
+ reader = (SegmentReader) IndexReader.Open(ram);
+ fi = reader.FieldInfos();
+ Assert.IsFalse(fi.FieldInfo("f1").StorePayloads_ForNUnitTest, "Payload field bit should not be set.");
+ Assert.IsTrue(fi.FieldInfo("f2").StorePayloads_ForNUnitTest, "Payload field bit should be set.");
+ Assert.IsTrue(fi.FieldInfo("f3").StorePayloads_ForNUnitTest, "Payload field bit should be set.");
+ reader.Close();
+ }
+
+ // Tests if payloads are correctly stored and loaded using both RamDirectory and FSDirectory
+ [Test]
+ public virtual void TestPayloadsEncoding()
+ {
+ // first perform the test using a RAMDirectory
+ Directory dir = new RAMDirectory();
+ PerformTest(dir);
+
+ // now use a FSDirectory and repeat same test
+ System.String dirName = "test_payloads";
+ dir = FSDirectory.GetDirectory(dirName);
+ PerformTest(dir);
+ RmDir(dirName);
+ }
+
+ // builds an index with payloads in the given Directory and performs
+ // different tests to verify the payload encoding
+ private void PerformTest(Directory dir)
+ {
+ PayloadAnalyzer analyzer = new PayloadAnalyzer();
+ IndexWriter writer = new IndexWriter(dir, analyzer, true);
+
+ // should be in sync with value in TermInfosWriter
+ int skipInterval = 16;
+
+ int numTerms = 5;
+ System.String fieldName = "f1";
+
+ int numDocs = skipInterval + 1;
+ // create content for the test documents with just a few terms
+ Term[] terms = GenerateTerms(fieldName, numTerms);
+ System.Text.StringBuilder sb = new System.Text.StringBuilder();
+ for (int i = 0; i < terms.Length; i++)
+ {
+ sb.Append(terms[i].text);
+ sb.Append(" ");
+ }
+ System.String content = sb.ToString();
+
+
+ int payloadDataLength = numTerms * numDocs * 2 + numTerms * numDocs * (numDocs - 1) / 2;
+ byte[] payloadData = GenerateRandomData(payloadDataLength);
+
+ Document d = new Document();
+ d.Add(new Field(fieldName, content, Field.Store.NO, Field.Index.TOKENIZED));
+ // add the same document multiple times to have the same payload lengths for all
+ // occurrences within two consecutive skip intervals
+ int offset = 0;
+ for (int i = 0; i < 2 * numDocs; i++)
+ {
+ analyzer.SetPayloadData(fieldName, payloadData, offset, 1);
+ offset += numTerms;
+ writer.AddDocument(d);
+ }
+
+ // make sure we create more than one segment to test merging
+ writer.Flush();
+
+ // now we make sure to have different payload lengths next at the next skip point
+ for (int i = 0; i < numDocs; i++)
+ {
+ analyzer.SetPayloadData(fieldName, payloadData, offset, i);
+ offset += i * numTerms;
+ writer.AddDocument(d);
+ }
+
+ writer.Optimize();
+ // flush
+ writer.Close();
+
+
+ /*
+ * Verify the index
+ * first we test if all payloads are stored correctly
+ */
+ IndexReader reader = IndexReader.Open(dir);
+
+ byte[] verifyPayloadData = new byte[payloadDataLength];
+ offset = 0;
+ TermPositions[] tps = new TermPositions[numTerms];
+ for (int i = 0; i < numTerms; i++)
+ {
+ tps[i] = reader.TermPositions(terms[i]);
+ }
+
+ while (tps[0].Next())
+ {
+ for (int i = 1; i < numTerms; i++)
+ {
+ tps[i].Next();
+ }
+ int freq = tps[0].Freq();
+
+ for (int i = 0; i < freq; i++)
+ {
+ for (int j = 0; j < numTerms; j++)
+ {
+ tps[j].NextPosition();
+ tps[j].GetPayload(verifyPayloadData, offset);
+ offset += tps[j].GetPayloadLength();
+ }
+ }
+ }
+
+ for (int i = 0; i < numTerms; i++)
+ {
+ tps[i].Close();
+ }
+
+ AssertByteArrayEquals(payloadData, verifyPayloadData);
+
+ /*
+ * test lazy skipping
+ */
+ TermPositions tp = reader.TermPositions(terms[0]);
+ tp.Next();
+ tp.NextPosition();
+ // now we don't read this payload
+ tp.NextPosition();
+ Assert.AreEqual(1, tp.GetPayloadLength(), "Wrong payload length.");
+ byte[] payload = tp.GetPayload(null, 0);
+ Assert.AreEqual(payload[0], payloadData[numTerms]);
+ tp.NextPosition();
+
+ // we don't read this payload and skip to a different document
+ tp.SkipTo(5);
+ tp.NextPosition();
+ Assert.AreEqual(1, tp.GetPayloadLength(), "Wrong payload length.");
+ payload = tp.GetPayload(null, 0);
+ Assert.AreEqual(payload[0], payloadData[5 * numTerms]);
+
+
+ /*
+ * Test different lengths at skip points
+ */
+ tp.Seek(terms[1]);
+ tp.Next();
+ tp.NextPosition();
+ Assert.AreEqual(1, tp.GetPayloadLength(), "Wrong payload length.");
+ tp.SkipTo(skipInterval - 1);
+ tp.NextPosition();
+ Assert.AreEqual(1, tp.GetPayloadLength(), "Wrong payload length.");
+ tp.SkipTo(2 * skipInterval - 1);
+ tp.NextPosition();
+ Assert.AreEqual(1, tp.GetPayloadLength(), "Wrong payload length.");
+ tp.SkipTo(3 * skipInterval - 1);
+ tp.NextPosition();
+ Assert.AreEqual(3 * skipInterval - 2 * numDocs - 1, tp.GetPayloadLength(), "Wrong payload length.");
+
+ /*
+ * Test multiple call of getPayload()
+ */
+ tp.GetPayload(null, 0);
+ try
+ {
+ // it is forbidden to call getPayload() more than once
+ // without calling nextPosition()
+ tp.GetPayload(null, 0);
+ Assert.Fail("Expected exception not thrown");
+ }
+ catch (System.Exception)
+ {
+ // expected exception
+ }
+
+ reader.Close();
+
+ // test long payload
+ analyzer = new PayloadAnalyzer();
+ writer = new IndexWriter(dir, analyzer, true);
+ System.String singleTerm = "lucene";
+
+ d = new Document();
+ d.Add(new Field(fieldName, singleTerm, Field.Store.NO, Field.Index.TOKENIZED));
+ // add a payload whose length is greater than the buffer size of BufferedIndexOutput
+ payloadData = GenerateRandomData(2000);
+ analyzer.SetPayloadData(fieldName, payloadData, 100, 1500);
+ writer.AddDocument(d);
+
+
+ writer.Optimize();
+ // flush
+ writer.Close();
+
+ reader = IndexReader.Open(dir);
+ tp = reader.TermPositions(new Term(fieldName, singleTerm));
+ tp.Next();
+ tp.NextPosition();
+
+ verifyPayloadData = new byte[tp.GetPayloadLength()];
+ tp.GetPayload(verifyPayloadData, 0);
+ byte[] portion = new byte[1500];
+ Array.Copy(payloadData, 100, portion, 0, 1500);
+
+ AssertByteArrayEquals(portion, verifyPayloadData);
+ reader.Close();
+ }
+
+ private static System.Random rnd = new System.Random();
+
+ private static void GenerateRandomData(byte[] data)
+ {
+ rnd.NextBytes(data);
+ }
+
+ private static byte[] GenerateRandomData(int n)
+ {
+ byte[] data = new byte[n];
+ GenerateRandomData(data);
+ return data;
+ }
+
+ private Term[] GenerateTerms(System.String fieldName, int n)
+ {
+ int maxDigits = (int) (System.Math.Log(n) / System.Math.Log(10));
+ Term[] terms = new Term[n];
+ System.Text.StringBuilder sb = new System.Text.StringBuilder();
+ for (int i = 0; i < n; i++)
+ {
+ sb.Length = 0;
+ sb.Append("t");
+ int zeros = maxDigits - (int) (System.Math.Log(i) / System.Math.Log(10));
+ for (int j = 0; j < zeros; j++)
+ {
+ sb.Append("0");
+ }
+ sb.Append(i);
+ terms[i] = new Term(fieldName, sb.ToString());
+ }
+ return terms;
+ }
+
+
+ private void RmDir(System.String dir)
+ {
+ System.IO.FileInfo fileDir = new System.IO.FileInfo(dir);
+ bool tmpBool;
+ if (System.IO.File.Exists(fileDir.FullName))
+ tmpBool = true;
+ else
+ tmpBool = System.IO.Directory.Exists(fileDir.FullName);
+ if (tmpBool)
+ {
+ System.IO.FileInfo[] files = SupportClass.FileSupport.GetFiles(fileDir);
+ if (files != null)
+ {
+ for (int i = 0; i < files.Length; i++)
+ {
+ bool tmpBool2;
+ if (System.IO.File.Exists(files[i].FullName))
+ {
+ System.IO.File.Delete(files[i].FullName);
+ tmpBool2 = true;
+ }
+ else if (System.IO.Directory.Exists(files[i].FullName))
+ {
+ System.IO.Directory.Delete(files[i].FullName);
+ tmpBool2 = true;
+ }
+ else
+ tmpBool2 = false;
+ bool generatedAux = tmpBool2;
+ }
+ }
+ bool tmpBool3;
+ if (System.IO.File.Exists(fileDir.FullName))
+ {
+ System.IO.File.Delete(fileDir.FullName);
+ tmpBool3 = true;
+ }
+ else if (System.IO.Directory.Exists(fileDir.FullName))
+ {
+ System.IO.Directory.Delete(fileDir.FullName);
+ tmpBool3 = true;
+ }
+ else
+ tmpBool3 = false;
+ bool generatedAux2 = tmpBool3;
+ }
+ }
+
+
+
+ internal virtual void AssertByteArrayEquals(byte[] b1, byte[] b2)
+ {
+ if (b1.Length != b2.Length)
+ {
+ Assert.Fail("Byte arrays have different lengths: " + b1.Length + ", " + b2.Length);
+ }
+
+ for (int i = 0; i < b1.Length; i++)
+ {
+ if (b1[i] != b2[i])
+ {
+ Assert.Fail("Byte arrays different at index " + i + ": " + b1[i] + ", " + b2[i]);
+ }
+ }
+ }
+
+
+ /// <summary> This Analyzer uses an WhitespaceTokenizer and PayloadFilter.</summary>
+ private class PayloadAnalyzer : Analyzer
+ {
+ internal System.Collections.IDictionary fieldToData = new System.Collections.Hashtable();
+
+ internal virtual void SetPayloadData(System.String field, byte[] data, int offset, int length)
+ {
+ fieldToData[field] = new PayloadData(0, data, offset, length);
+ }
+
+ internal virtual void SetPayloadData(System.String field, int numFieldInstancesToSkip, byte[] data, int offset, int length)
+ {
+ fieldToData[field] = new PayloadData(numFieldInstancesToSkip, data, offset, length);
+ }
+
+ public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
+ {
+ PayloadData payload = (PayloadData) fieldToData[fieldName];
+ TokenStream ts = new WhitespaceTokenizer(reader);
+ if (payload != null)
+ {
+ if (payload.numFieldInstancesToSkip == 0)
+ {
+ ts = new PayloadFilter(ts, payload.data, payload.offset, payload.length);
+ }
+ else
+ {
+ payload.numFieldInstancesToSkip--;
+ }
+ }
+ return ts;
+ }
+
+ private class PayloadData
+ {
+ internal byte[] data;
+ internal int offset;
+ internal int length;
+ internal int numFieldInstancesToSkip;
+
+ internal PayloadData(int skip, byte[] data, int offset, int length)
+ {
+ numFieldInstancesToSkip = skip;
+ this.data = data;
+ this.offset = offset;
+ this.length = length;
+ }
+ }
+ }
+
+
+ /// <summary> This Filter adds payloads to the tokens.</summary>
+ private class PayloadFilter : TokenFilter
+ {
+ private byte[] data;
+ private int length;
+ private int offset;
+ internal Payload payload = new Payload();
+
+ public PayloadFilter(TokenStream in_Renamed, byte[] data, int offset, int length):base(in_Renamed)
+ {
+ this.data = data;
+ this.length = length;
+ this.offset = offset;
+ }
+
+ public override Token Next(Token token)
+ {
+ token = input.Next(token);
+ if (token != null)
+ {
+ if (offset + length <= data.Length)
+ {
+ Payload p = null;
+ if (p == null)
+ {
+ p = new Payload();
+ token.SetPayload(p);
+ }
+ p.SetData(data, offset, length);
+ offset += length;
+ }
+ else
+ {
+ token.SetPayload(null);
+ }
+ }
+
+ return token;
+ }
+ }
+
+ [Test]
+ public virtual void TestThreadSafety()
+ {
+ int numThreads = 5;
+ int numDocs = 50;
+ ByteArrayPool pool = new ByteArrayPool(numThreads, 5);
+
+ Directory dir = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer());
+ System.String field = "test";
+
+ SupportClass.ThreadClass[] ingesters = new SupportClass.ThreadClass[numThreads];
+ for (int i = 0; i < numThreads; i++)
+ {
+ ingesters[i] = new AnonymousClassThread(numDocs, field, pool, writer, this);
+ ingesters[i].Start();
+ }
+
+ for (int i = 0; i < numThreads; i++)
+ {
+ try
+ {
+ ingesters[i].Join();
+ }
+ catch (System.Threading.ThreadInterruptedException)
+ {
+ }
+ }
+ writer.Close();
+ IndexReader reader = IndexReader.Open(dir);
+ TermEnum terms = reader.Terms();
+ while (terms.Next())
+ {
+ TermPositions tp = reader.TermPositions(terms.Term());
+ while (tp.Next())
+ {
+ int freq = tp.Freq();
+ for (int i = 0; i < freq; i++)
+ {
+ tp.NextPosition();
+ System.String s = System.Text.Encoding.UTF8.GetString(tp.GetPayload(new byte[5], 0));
+ Assert.AreEqual(s, terms.Term().text);
+ }
+ }
+ tp.Close();
+ }
+ terms.Close();
+ reader.Close();
+
+ Assert.AreEqual(pool.Size(), numThreads);
+ }
+
+ private class PoolingPayloadTokenStream : TokenStream
+ {
+ private byte[] payload;
+ private bool first;
+ private ByteArrayPool pool;
+
+ internal PoolingPayloadTokenStream(ByteArrayPool pool)
+ {
+ this.pool = pool;
+ payload = pool.Get();
+ Lucene.Net.Index.TestPayloads.GenerateRandomData(payload);
+ first = true;
+ }
+
+ public override Token Next()
+ {
+ if (!first)
+ return null;
+ Token t = new Token(System.Text.Encoding.UTF8.GetString(payload), 0, 0);
+ t.SetPayload(new Payload(payload));
+ return t;
+ }
+
+ public override void Close()
+ {
+ pool.Release(payload);
+ }
+ }
+
+ internal class ByteArrayPool
+ {
+ private System.Collections.IList pool;
+
+ internal ByteArrayPool(int capacity, int size)
+ {
+ pool = new System.Collections.ArrayList();
+ for (int i = 0; i < capacity; i++)
+ {
+ pool.Add(new byte[size]);
+ }
+ }
+
+ internal virtual byte[] Get()
+ {
+ lock (this)
+ {
+ System.Object tempObject;
+ tempObject = pool[0];
+ pool.RemoveAt(0);
+ return (byte[]) tempObject;
+ }
+ }
+
+ internal virtual void Release(byte[] b)
+ {
+ lock (this)
+ {
+ pool.Add(b);
+ }
+ }
+
+ internal virtual int Size()
+ {
+ lock (this)
+ {
+ return pool.Count;
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestPositionBasedTermVectorMapper.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestPositionBasedTermVectorMapper.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestPositionBasedTermVectorMapper.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestPositionBasedTermVectorMapper.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using NUnit.Framework;
+
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+namespace Lucene.Net.Index
+{
+
+ [TestFixture]
+ public class TestPositionBasedTermVectorMapper : LuceneTestCase
+ {
+ protected internal System.String[] tokens;
+ protected internal int[][] thePositions;
+ protected internal TermVectorOffsetInfo[][] offsets;
+ protected internal int numPositions;
+
+
+ //public TestPositionBasedTermVectorMapper(System.String s):base(s)
+ //{
+ //}
+
+ [SetUp]
+ public override void SetUp()
+ {
+ base.SetUp();
+ tokens = new System.String[]{"here", "is", "some", "text", "to", "test", "extra"};
+ thePositions = new int[tokens.Length][];
+ offsets = new TermVectorOffsetInfo[tokens.Length][];
+ numPositions = 0;
+ //save off the last one so we can add it with the same positions as some of the others, but in a predictable way
+ for (int i = 0; i < tokens.Length - 1; i++)
+ {
+ thePositions[i] = new int[2 * i + 1]; //give 'em all some positions
+ for (int j = 0; j < thePositions[i].Length; j++)
+ {
+ thePositions[i][j] = numPositions++;
+ }
+ offsets[i] = new TermVectorOffsetInfo[thePositions[i].Length];
+ for (int j = 0; j < offsets[i].Length; j++)
+ {
+ offsets[i][j] = new TermVectorOffsetInfo(j, j + 1); //the actual value here doesn't much matter
+ }
+ }
+ thePositions[tokens.Length - 1] = new int[1];
+ thePositions[tokens.Length - 1][0] = 0; //put this at the same position as "here"
+ offsets[tokens.Length - 1] = new TermVectorOffsetInfo[1];
+ offsets[tokens.Length - 1][0] = new TermVectorOffsetInfo(0, 1);
+ }
+
+ [Test]
+ public virtual void Test()
+ {
+ PositionBasedTermVectorMapper mapper = new PositionBasedTermVectorMapper();
+
+ mapper.SetExpectations("test", tokens.Length, true, true);
+ //Test single position
+ for (int i = 0; i < tokens.Length; i++)
+ {
+ System.String token = tokens[i];
+ mapper.Map(token, 1, null, thePositions[i]);
+ }
+ System.Collections.IDictionary map = mapper.GetFieldToTerms();
+ Assert.IsTrue(map != null, "map is null and it shouldn't be");
+ Assert.IsTrue(map.Count == 1, "map Size: " + map.Count + " is not: " + 1);
+ System.Collections.IDictionary positions = (System.Collections.IDictionary) map["test"];
+ Assert.IsTrue(positions != null, "thePositions is null and it shouldn't be");
+
+ Assert.IsTrue(positions.Count == numPositions, "thePositions Size: " + positions.Count + " is not: " + numPositions);
+ System.Collections.BitArray bits = new System.Collections.BitArray((numPositions % 64 == 0?numPositions / 64:numPositions / 64 + 1) * 64);
+ for (System.Collections.IEnumerator iterator = new System.Collections.Hashtable(positions).GetEnumerator(); iterator.MoveNext(); )
+ {
+ System.Collections.DictionaryEntry entry = (System.Collections.DictionaryEntry) iterator.Current;
+ PositionBasedTermVectorMapper.TVPositionInfo info = (PositionBasedTermVectorMapper.TVPositionInfo) entry.Value;
+ Assert.IsTrue(info != null, "info is null and it shouldn't be");
+ int pos = ((System.Int32) entry.Key);
+ bits.Set(pos, true);
+ Assert.IsTrue(info.GetPosition() == pos, info.GetPosition() + " does not equal: " + pos);
+ Assert.IsTrue(info.GetOffsets() != null, "info.getOffsets() is null and it shouldn't be");
+ if (pos == 0)
+ {
+ Assert.IsTrue(info.GetTerms().Count == 2, "info.getTerms() Size: " + info.GetTerms().Count + " is not: " + 2); //need a test for multiple terms at one pos
+ Assert.IsTrue(info.GetOffsets().Count == 2, "info.getOffsets() Size: " + info.GetOffsets().Count + " is not: " + 2);
+ }
+ else
+ {
+ Assert.IsTrue(info.GetTerms().Count == 1, "info.getTerms() Size: " + info.GetTerms().Count + " is not: " + 1); //need a test for multiple terms at one pos
+ Assert.IsTrue(info.GetOffsets().Count == 1, "info.getOffsets() Size: " + info.GetOffsets().Count + " is not: " + 1);
+ }
+ }
+ int cardinality = 0;
+ for (int i = 0; i < bits.Count; i++)
+ {
+ if (bits.Get(i)) cardinality++;
+ }
+ Assert.IsTrue(cardinality == numPositions, "Bits are not all on");
+ }
+ }
+}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentMerger.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestSegmentMerger.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentMerger.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentMerger.cs Tue Jul 15 14:44:04 2008
@@ -19,70 +19,64 @@
using NUnit.Framework;
+using Document = Lucene.Net.Documents.Document;
using Directory = Lucene.Net.Store.Directory;
using RAMDirectory = Lucene.Net.Store.RAMDirectory;
-using Document = Lucene.Net.Documents.Document;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Index
{
[TestFixture]
- public class TestSegmentMerger
+ public class TestSegmentMerger : LuceneTestCase
{
//The variables for the new merged segment
private Directory mergedDir = new RAMDirectory();
private System.String mergedSegment = "test";
//First segment to be merged
private Directory merge1Dir = new RAMDirectory();
- private Lucene.Net.Documents.Document doc1 = new Lucene.Net.Documents.Document();
- private System.String merge1Segment = "test-1";
+ private Document doc1 = new Document();
private SegmentReader reader1 = null;
//Second Segment to be merged
private Directory merge2Dir = new RAMDirectory();
- private Lucene.Net.Documents.Document doc2 = new Lucene.Net.Documents.Document();
- private System.String merge2Segment = "test-2";
+ private Document doc2 = new Document();
private SegmentReader reader2 = null;
-
-
- // public TestSegmentMerger(System.String s)
- // {
- // }
-
- // This is needed if for the test to pass and mimic what happens wiht JUnit
- // For some reason, JUnit is creating a new member variable for each sub-test
- // but NUnit is not -- who is wrong/right, I don't know.
- private void SetUpInternal() // {{Aroush-1.9}} See note above
- {
- //The variables for the new merged segment
- mergedDir = new RAMDirectory();
- mergedSegment = "test";
- //First segment to be merged
- merge1Dir = new RAMDirectory();
- doc1 = new Lucene.Net.Documents.Document();
- merge1Segment = "test-1";
- reader1 = null;
- //Second Segment to be merged
- merge2Dir = new RAMDirectory();
- doc2 = new Lucene.Net.Documents.Document();
- merge2Segment = "test-2";
- reader2 = null;
- }
- [SetUp]
- public virtual void SetUp()
+ // This is needed if for the test to pass and mimic what happens wiht JUnit
+ // For some reason, JUnit is creating a new member variable for each sub-test
+ // but NUnit is not -- who is wrong/right, I don't know.
+ private void SetUpInternal() // {{Aroush-1.9}} See note above
{
- SetUpInternal(); // We need this for NUnit; see note above
+ //The variables for the new merged segment
+ mergedDir = new RAMDirectory();
+ mergedSegment = "test";
+ //First segment to be merged
+ merge1Dir = new RAMDirectory();
+ doc1 = new Lucene.Net.Documents.Document();
+ //merge1Segment = "test-1";
+ reader1 = null;
+ //Second Segment to be merged
+ merge2Dir = new RAMDirectory();
+ doc2 = new Lucene.Net.Documents.Document();
+ //merge2Segment = "test-2";
+ reader2 = null;
+ }
+ [SetUp]
+ public override void SetUp()
+ {
+ base.SetUp();
+ SetUpInternal();
DocHelper.SetupDoc(doc1);
- DocHelper.WriteDoc(merge1Dir, merge1Segment, doc1);
+ SegmentInfo info1 = DocHelper.WriteDoc(merge1Dir, doc1);
DocHelper.SetupDoc(doc2);
- DocHelper.WriteDoc(merge2Dir, merge2Segment, doc2);
- reader1 = SegmentReader.Get(new SegmentInfo(merge1Segment, 1, merge1Dir));
- reader2 = SegmentReader.Get(new SegmentInfo(merge2Segment, 1, merge2Dir));
+ SegmentInfo info2 = DocHelper.WriteDoc(merge2Dir, doc2);
+ reader1 = SegmentReader.Get(info1);
+ reader2 = SegmentReader.Get(info2);
}
[Test]
- public virtual void Test()
+ public virtual void Test()
{
Assert.IsTrue(mergedDir != null);
Assert.IsTrue(merge1Dir != null);
@@ -92,7 +86,7 @@
}
[Test]
- public virtual void TestMerge()
+ public virtual void TestMerge()
{
SegmentMerger merger = new SegmentMerger(mergedDir, mergedSegment);
merger.Add(reader1);
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestSegmentReader.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentReader.cs Tue Jul 15 14:44:04 2008
@@ -21,52 +21,56 @@
using Document = Lucene.Net.Documents.Document;
using Fieldable = Lucene.Net.Documents.Fieldable;
-using DefaultSimilarity = Lucene.Net.Search.DefaultSimilarity;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+using DefaultSimilarity = Lucene.Net.Search.DefaultSimilarity;
+using Similarity = Lucene.Net.Search.Similarity;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Index
{
[TestFixture]
- public class TestSegmentReader
+ public class TestSegmentReader : LuceneTestCase
{
private RAMDirectory dir = new RAMDirectory();
private Lucene.Net.Documents.Document testDoc = new Lucene.Net.Documents.Document();
private SegmentReader reader = null;
- // public TestSegmentReader(System.String s)
- // {
- // }
-
- // This is needed if for the test to pass and mimic what happens wiht JUnit
- // For some reason, JUnit is creating a new member variable for each sub-test
- // but NUnit is not -- who is wrong/right, I don't know.
- private void SetUpInternal() // {{Aroush-1.9}} See note above
- {
- dir = new RAMDirectory();
- testDoc = new Lucene.Net.Documents.Document();
- reader = null;
- }
+ // public TestSegmentReader(System.String s)
+ // {
+ // }
+
+ // This is needed if for the test to pass and mimic what happens wiht JUnit
+ // For some reason, JUnit is creating a new member variable for each sub-test
+ // but NUnit is not -- who is wrong/right, I don't know.
+ private void SetUpInternal() // {{Aroush-1.9}} See note above
+ {
+ dir = new RAMDirectory();
+ testDoc = new Lucene.Net.Documents.Document();
+ reader = null;
+ }
//TODO: Setup the reader w/ multiple documents
[SetUp]
- public virtual void SetUp()
+ public override void SetUp()
{
- SetUpInternal(); // We need this for NUnit; see note above
-
+ base.SetUp();
+ SetUpInternal();
DocHelper.SetupDoc(testDoc);
- DocHelper.WriteDoc(dir, testDoc);
- reader = SegmentReader.Get(new SegmentInfo("test", 1, dir));
+ SegmentInfo info = DocHelper.WriteDoc(dir, testDoc);
+ reader = SegmentReader.Get(info);
}
[TearDown]
- public virtual void TearDown()
+ public override void TearDown()
{
}
[Test]
- public virtual void Test()
+ public virtual void Test()
{
Assert.IsTrue(dir != null);
Assert.IsTrue(reader != null);
@@ -75,29 +79,31 @@
}
[Test]
- public virtual void TestDocument()
+ public virtual void TestDocument()
{
+ int i = 1;
Assert.IsTrue(reader.NumDocs() == 1);
Assert.IsTrue(reader.MaxDoc() >= 1);
Lucene.Net.Documents.Document result = reader.Document(0);
Assert.IsTrue(result != null);
//There are 2 unstored fields on the document that are not preserved across writing
Assert.IsTrue(DocHelper.NumFields(result) == DocHelper.NumFields(testDoc) - DocHelper.unstored.Count);
-
- foreach (Lucene.Net.Documents.Field field in result.Fields())
+ System.Collections.IEnumerator e = result.Fields();
+ while (e.MoveNext())
{
+ Lucene.Net.Documents.Field field = (Lucene.Net.Documents.Field) e.Current;
Assert.IsTrue(field != null);
Assert.IsTrue(DocHelper.nameValues.Contains(field.Name()));
}
}
[Test]
- public virtual void TestDelete()
+ public virtual void TestDelete()
{
Lucene.Net.Documents.Document docToDelete = new Lucene.Net.Documents.Document();
DocHelper.SetupDoc(docToDelete);
- DocHelper.WriteDoc(dir, "seg-to-delete", docToDelete);
- SegmentReader deleteReader = SegmentReader.Get(new SegmentInfo("seg-to-delete", 1, dir));
+ SegmentInfo info = DocHelper.WriteDoc(dir, docToDelete);
+ SegmentReader deleteReader = SegmentReader.Get(info);
Assert.IsTrue(deleteReader != null);
Assert.IsTrue(deleteReader.NumDocs() == 1);
deleteReader.DeleteDocument(0);
@@ -109,21 +115,21 @@
deleteReader.Document(0);
Assert.Fail();
}
- catch (System.ArgumentException e)
+ catch (System.ArgumentException)
{
// expcected exception
}
}
[Test]
- public virtual void TestGetFieldNameVariations()
+ public virtual void TestGetFieldNameVariations()
{
System.Collections.ICollection result = reader.GetFieldNames(IndexReader.FieldOption.ALL);
Assert.IsTrue(result != null);
Assert.IsTrue(result.Count == DocHelper.all.Count);
for (System.Collections.IEnumerator iter = result.GetEnumerator(); iter.MoveNext(); )
{
- System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iter.Current;
+ System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iter.Current;
System.String s = fi.Key.ToString();
//System.out.println("Name: " + s);
Assert.IsTrue(DocHelper.nameValues.Contains(s) == true || s.Equals(""));
@@ -133,9 +139,9 @@
Assert.IsTrue(result.Count == DocHelper.indexed.Count);
for (System.Collections.IEnumerator iter = result.GetEnumerator(); iter.MoveNext(); )
{
- System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iter.Current;
- System.String s = fi.Key.ToString();
- Assert.IsTrue(DocHelper.indexed.Contains(s) == true || s.Equals(""));
+ System.Collections.DictionaryEntry fi = (System.Collections.DictionaryEntry) iter.Current;
+ System.String s = fi.Key.ToString();
+ Assert.IsTrue(DocHelper.indexed.Contains(s) == true || s.Equals(""));
}
result = reader.GetFieldNames(IndexReader.FieldOption.UNINDEXED);
@@ -152,7 +158,7 @@
}
[Test]
- public virtual void TestTerms()
+ public virtual void TestTerms()
{
TermEnum terms = reader.Terms();
Assert.IsTrue(terms != null);
@@ -182,7 +188,7 @@
}
[Test]
- public virtual void TestNorms()
+ public virtual void TestNorms()
{
//TODO: Not sure how these work/should be tested
/*
@@ -203,7 +209,7 @@
// test omit norms
for (int i = 0; i < DocHelper.fields.Length; i++)
{
- Lucene.Net.Documents.Field f = DocHelper.fields[i];
+ Lucene.Net.Documents.Fieldable f = DocHelper.fields[i];
if (f.IsIndexed())
{
Assert.AreEqual(reader.HasNorms(f.Name()), !f.GetOmitNorms());
@@ -247,5 +253,23 @@
Assert.IsTrue(results != null);
Assert.IsTrue(results.Length == 4, "We do not have 4 term freq vectors, we have: " + results.Length);
}
+
+ [Test]
+ public virtual void TestIndexDivisor()
+ {
+ dir = new MockRAMDirectory();
+ testDoc = new Document();
+ DocHelper.SetupDoc(testDoc);
+ SegmentInfo si = DocHelper.WriteDoc(dir, testDoc);
+
+ reader = SegmentReader.Get(si);
+ reader.SetTermInfosIndexDivisor(3);
+ TestDocument();
+ TestDelete();
+ TestGetFieldNameVariations();
+ TestNorms();
+ TestTerms();
+ TestTermVectors();
+ }
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermDocs.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestSegmentTermDocs.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermDocs.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermDocs.cs Tue Jul 15 14:44:04 2008
@@ -19,60 +19,62 @@
using NUnit.Framework;
-using RAMDirectory = Lucene.Net.Store.RAMDirectory;
-using Directory = Lucene.Net.Store.Directory;
-using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
using Document = Lucene.Net.Documents.Document;
using Field = Lucene.Net.Documents.Field;
+using Directory = Lucene.Net.Store.Directory;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
+using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+using Similarity = Lucene.Net.Search.Similarity;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Index
{
[TestFixture]
- public class TestSegmentTermDocs
+ public class TestSegmentTermDocs : LuceneTestCase
{
private Lucene.Net.Documents.Document testDoc = new Lucene.Net.Documents.Document();
private Directory dir = new RAMDirectory();
+ private SegmentInfo info;
- // public TestSegmentTermDocs(System.String s)
- // {
- // }
-
- // This is needed if for the test to pass and mimic what happens wiht JUnit
- // For some reason, JUnit is creating a new member variable for each sub-test
- // but NUnit is not -- who is wrong/right, I don't know.
- private void SetUpInternal() // {{Aroush-1.9}} See note above
- {
- testDoc = new Lucene.Net.Documents.Document();
- dir = new RAMDirectory();
- }
+ // got the idea for this from George's note in TestSegmentReader
+ // it seems that JUnit creates a new instance of the class for each test invocation
+ // while NUnit does not (seems like a flaw in JUnit, to be honest)
+ // forcing the re-init of the variables for each run solves the problem
+ private void SetUpInternal()
+ {
+ dir = new RAMDirectory();
+ testDoc = new Lucene.Net.Documents.Document();
+ info = null;
+ }
[SetUp]
- public virtual void SetUp()
+ public override void SetUp()
{
- SetUpInternal(); // We need this for NUnit; see note above
-
+ base.SetUp();
+ SetUpInternal();
DocHelper.SetupDoc(testDoc);
- DocHelper.WriteDoc(dir, testDoc);
+ info = DocHelper.WriteDoc(dir, testDoc);
}
- [TearDown]
- public virtual void TearDown()
+ [Test]
+ public virtual void Test()
{
-
+ Assert.IsTrue(dir != null);
}
[Test]
- public virtual void Test()
+ public virtual void TestTermDocs()
{
- Assert.IsTrue(dir != null);
+ TestTermDocs(1);
}
- [Test]
- public virtual void TestTermDocs()
+ public virtual void TestTermDocs(int indexDivisor)
{
//After adding the document, we should be able to read it back in
- SegmentReader reader = SegmentReader.Get(new SegmentInfo("test", 1, dir));
+ SegmentReader reader = SegmentReader.Get(info);
+ reader.SetTermInfosIndexDivisor(indexDivisor);
Assert.IsTrue(reader != null);
SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
Assert.IsTrue(segTermDocs != null);
@@ -88,11 +90,17 @@
}
[Test]
- public virtual void TestBadSeek()
+ public virtual void TestBadSeek()
+ {
+ TestBadSeek(1);
+ }
+
+ public virtual void TestBadSeek(int indexDivisor)
{
{
//After adding the document, we should be able to read it back in
- SegmentReader reader = SegmentReader.Get(new SegmentInfo("test", 1, dir));
+ SegmentReader reader = SegmentReader.Get(info);
+ reader.SetTermInfosIndexDivisor(indexDivisor);
Assert.IsTrue(reader != null);
SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
Assert.IsTrue(segTermDocs != null);
@@ -102,7 +110,8 @@
}
{
//After adding the document, we should be able to read it back in
- SegmentReader reader = SegmentReader.Get(new SegmentInfo("test", 1, dir));
+ SegmentReader reader = SegmentReader.Get(info);
+ reader.SetTermInfosIndexDivisor(indexDivisor);
Assert.IsTrue(reader != null);
SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
Assert.IsTrue(segTermDocs != null);
@@ -113,7 +122,12 @@
}
[Test]
- public virtual void TestSkipTo()
+ public virtual void TestSkipTo()
+ {
+ TestSkipTo(1);
+ }
+
+ public virtual void TestSkipTo(int indexDivisor)
{
Directory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
@@ -135,6 +149,9 @@
writer.Close();
IndexReader reader = IndexReader.Open(dir);
+ reader.SetTermInfosIndexDivisor(indexDivisor);
+ Assert.AreEqual(indexDivisor, reader.GetTermInfosIndexDivisor());
+
TermDocs tdocs = reader.TermDocs();
// without optimization (assumption skipInterval == 16)
@@ -238,6 +255,38 @@
dir.Close();
}
+ [Test]
+ public virtual void TestIndexDivisor()
+ {
+ dir = new MockRAMDirectory();
+ testDoc = new Document();
+ DocHelper.SetupDoc(testDoc);
+ DocHelper.WriteDoc(dir, testDoc);
+ TestTermDocs(2);
+ TestBadSeek(2);
+ TestSkipTo(2);
+ }
+
+ [Test]
+ public virtual void TestIndexDivisorAfterLoad()
+ {
+ dir = new MockRAMDirectory();
+ testDoc = new Document();
+ DocHelper.SetupDoc(testDoc);
+ SegmentInfo si = DocHelper.WriteDoc(dir, testDoc);
+ SegmentReader reader = SegmentReader.Get(si);
+ Assert.AreEqual(1, reader.DocFreq(new Term("keyField", "Keyword")));
+ try
+ {
+ reader.SetTermInfosIndexDivisor(2);
+ Assert.Fail("did not hit IllegalStateException exception");
+ }
+ catch (System.SystemException)
+ {
+ // expected
+ }
+ }
+
private void AddDoc(IndexWriter writer, System.String value_Renamed)
{
Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermEnum.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestSegmentTermEnum.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermEnum.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestSegmentTermEnum.cs Tue Jul 15 14:44:04 2008
@@ -19,11 +19,13 @@
using NUnit.Framework;
-using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
using Document = Lucene.Net.Documents.Document;
using Field = Lucene.Net.Documents.Field;
using Directory = Lucene.Net.Store.Directory;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
namespace Lucene.Net.Index
{
@@ -31,12 +33,12 @@
/// <author> goller
/// </author>
[TestFixture]
- public class TestSegmentTermEnum
+ public class TestSegmentTermEnum : LuceneTestCase
{
internal Directory dir = new RAMDirectory();
[Test]
- public virtual void TestTermEnum()
+ public virtual void TestTermEnum()
{
IndexWriter writer = null;
@@ -65,6 +67,24 @@
VerifyDocFreq();
}
+ [Test]
+ public virtual void TestPrevTermAtEnd()
+ {
+ Directory dir = new MockRAMDirectory();
+ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+ AddDoc(writer, "aaa bbb");
+ writer.Close();
+ IndexReader reader = IndexReader.Open(dir);
+ SegmentTermEnum termEnum = (SegmentTermEnum) reader.Terms();
+ Assert.IsTrue(termEnum.Next());
+ Assert.AreEqual("aaa", termEnum.Term().Text());
+ Assert.IsTrue(termEnum.Next());
+ Assert.AreEqual("aaa", termEnum.Prev().Text());
+ Assert.AreEqual("bbb", termEnum.Term().Text());
+ Assert.IsFalse(termEnum.Next());
+ Assert.AreEqual("bbb", termEnum.Prev().Text());
+ }
+
private void VerifyDocFreq()
{
IndexReader reader = IndexReader.Open(dir);
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestStressIndexing.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing.cs Tue Jul 15 14:44:04 2008
@@ -19,148 +19,165 @@
using NUnit.Framework;
-using Lucene.Net.Util;
-using Lucene.Net.Store;
using Lucene.Net.Documents;
-using Lucene.Net.Analysis;
using Lucene.Net.Index;
+using Lucene.Net.QueryParsers;
+using Lucene.Net.Store;
+using Lucene.Net.Util;
+using Lucene.Net.Analysis;
using Lucene.Net.Search;
using Searchable = Lucene.Net.Search.Searchable;
-using Lucene.Net.QueryParsers;
namespace Lucene.Net.Index
{
- [TestFixture]
- public class TestStressIndexing
+ [TestFixture]
+ public class TestStressIndexing : LuceneTestCase
{
private static readonly Analyzer ANALYZER = new SimpleAnalyzer();
private static readonly System.Random RANDOM = new System.Random();
- private static Searcher SEARCHER;
-
- private static int RUN_TIME_SEC = 15;
- private class IndexerThread : SupportClass.ThreadClass
+ abstract public class TimedThread : SupportClass.ThreadClass
{
- internal IndexModifier modifier;
- internal int nextID;
- public int count;
internal bool failed;
+ internal int count;
+ private static int RUN_TIME_SEC = 6;
+ private TimedThread[] allThreads;
+
+ abstract public void DoWork();
- public IndexerThread(IndexModifier modifier)
+ internal TimedThread(TimedThread[] threads)
{
- this.modifier = modifier;
+ this.allThreads = threads;
}
override public void Run()
{
- long stopTime = (System.DateTime.Now.Ticks - 621355968000000000) / 10000 + 1000 * Lucene.Net.Index.TestStressIndexing.RUN_TIME_SEC;
+ long stopTime = (System.DateTime.Now.Ticks - 621355968000000000) / 10000 + 1000 * RUN_TIME_SEC;
+
+ count = 0;
+
try
{
- while (true)
+ while ((System.DateTime.Now.Ticks - 621355968000000000) / 10000 < stopTime && !AnyErrors())
{
-
- if ((System.DateTime.Now.Ticks - 621355968000000000) / 10000 > stopTime)
- {
- break;
- }
-
- // Add 10 docs:
- for (int j = 0; j < 10; j++)
- {
- Lucene.Net.Documents.Document d = new Lucene.Net.Documents.Document();
- int n = Lucene.Net.Index.TestStressIndexing.RANDOM.Next();
- d.Add(new Field("id", System.Convert.ToString(nextID++), Field.Store.YES, Field.Index.UN_TOKENIZED));
- d.Add(new Field("contents", English.IntToEnglish(n), Field.Store.NO, Field.Index.TOKENIZED));
- modifier.AddDocument(d);
- }
-
- // Delete 5 docs:
- int deleteID = nextID;
- for (int j = 0; j < 5; j++)
- {
- modifier.DeleteDocuments(new Term("id", "" + deleteID));
- deleteID -= 2;
- }
-
+ DoWork();
count++;
}
-
- modifier.Close();
}
catch (System.Exception e)
{
- System.Console.Out.WriteLine(e.ToString());
- System.Console.Error.WriteLine(e.StackTrace);
+ System.Console.Out.WriteLine(e.StackTrace);
failed = true;
}
}
+
+ private bool AnyErrors()
+ {
+ for (int i = 0; i < allThreads.Length; i++)
+ if (allThreads[i] != null && allThreads[i].failed)
+ return true;
+ return false;
+ }
}
- private class SearcherThread : SupportClass.ThreadClass
+ private class IndexerThread : TimedThread
{
- private Directory directory;
- public int count;
- internal bool failed;
+ internal IndexWriter writer;
+ //new public int count;
+ internal int nextID;
- public SearcherThread(Directory directory)
+ public IndexerThread(IndexWriter writer, TimedThread[] threads):base(threads)
{
- this.directory = directory;
+ this.writer = writer;
}
- override public void Run()
+ public override void DoWork()
{
- long stopTime = (System.DateTime.Now.Ticks - 621355968000000000) / 10000 + 1000 * Lucene.Net.Index.TestStressIndexing.RUN_TIME_SEC;
- try
+ // Add 10 docs:
+ for (int j = 0; j < 10; j++)
{
- while (true)
- {
- for (int i = 0; i < 100; i++)
- {
- (new IndexSearcher(directory)).Close();
- }
- count += 100;
- if ((System.DateTime.Now.Ticks - 621355968000000000) / 10000 > stopTime)
- {
- break;
- }
- }
+ Document d = new Document();
+ int n = Lucene.Net.Index.TestStressIndexing.RANDOM.Next();
+ d.Add(new Field("id", System.Convert.ToString(nextID++), Field.Store.YES, Field.Index.UN_TOKENIZED));
+ d.Add(new Field("contents", English.IntToEnglish(n), Field.Store.NO, Field.Index.TOKENIZED));
+ writer.AddDocument(d);
}
- catch (System.Exception e)
+
+ // Delete 5 docs:
+ int deleteID = nextID - 1;
+ for (int j = 0; j < 5; j++)
{
- System.Console.Out.WriteLine(e.ToString());
- System.Console.Error.WriteLine(e.StackTrace);
- failed = true;
+ writer.DeleteDocuments(new Term("id", "" + deleteID));
+ deleteID -= 2;
}
}
}
+ private class SearcherThread : TimedThread
+ {
+ private Directory directory;
+
+ public SearcherThread(Directory directory, TimedThread[] threads):base(threads)
+ {
+ this.directory = directory;
+ }
+
+ public override void DoWork()
+ {
+ for (int i = 0; i < 100; i++)
+ (new IndexSearcher(directory)).Close();
+ count += 100;
+ }
+ }
+
/*
Run one indexer and 2 searchers against single index as
stress test.
*/
- public virtual void RunStressTest(Directory directory)
+ public virtual void RunStressTest(Directory directory, bool autoCommit, MergeScheduler mergeScheduler)
{
- IndexModifier modifier = new IndexModifier(directory, ANALYZER, true);
+ IndexWriter modifier = new IndexWriter(directory, autoCommit, ANALYZER, true);
+
+ modifier.SetMaxBufferedDocs(10);
+
+ TimedThread[] threads = new TimedThread[4];
+
+ if (mergeScheduler != null)
+ modifier.SetMergeScheduler(mergeScheduler);
// One modifier that writes 10 docs then removes 5, over
// and over:
- IndexerThread indexerThread = new IndexerThread(modifier);
+ IndexerThread indexerThread = new IndexerThread(modifier, threads);
+ threads[0] = indexerThread;
indexerThread.Start();
- // Two searchers that constantly just re-instantiate the searcher:
- SearcherThread searcherThread1 = new SearcherThread(directory);
+ IndexerThread indexerThread2 = new IndexerThread(modifier, threads);
+ threads[2] = indexerThread2;
+ indexerThread2.Start();
+
+ // Two searchers that constantly just re-instantiate the
+ // searcher:
+ SearcherThread searcherThread1 = new SearcherThread(directory, threads);
+ threads[3] = searcherThread1;
searcherThread1.Start();
- SearcherThread searcherThread2 = new SearcherThread(directory);
+ SearcherThread searcherThread2 = new SearcherThread(directory, threads);
+ threads[3] = searcherThread2;
searcherThread2.Start();
indexerThread.Join();
+ indexerThread2.Join();
searcherThread1.Join();
searcherThread2.Join();
- Assert.IsTrue(!indexerThread.failed,"hit unexpected exception in indexer");
- Assert.IsTrue(!searcherThread1.failed,"hit unexpected exception in search1");
+
+ modifier.Close();
+
+ Assert.IsTrue(!indexerThread.failed, "hit unexpected exception in indexer");
+ Assert.IsTrue(!indexerThread2.failed, "hit unexpected exception in indexer2");
+ Assert.IsTrue(!searcherThread1.failed, "hit unexpected exception in search1");
Assert.IsTrue(!searcherThread2.failed, "hit unexpected exception in search2");
+
//System.out.println(" Writer: " + indexerThread.count + " iterations");
//System.out.println("Searcher 1: " + searcherThread1.count + " searchers created");
//System.out.println("Searcher 2: " + searcherThread2.count + " searchers created");
@@ -170,58 +187,43 @@
Run above stress test against RAMDirectory and then
FSDirectory.
*/
- [Test]
+ [Test]
public virtual void TestStressIndexAndSearching()
{
- // First in a RAM directory:
- Directory directory = new RAMDirectory();
- RunStressTest(directory);
+ // RAMDir
+ Directory directory = new MockRAMDirectory();
+ RunStressTest(directory, true, null);
directory.Close();
- // Second in an FSDirectory:
+ // FSDir
System.String tempDir = System.IO.Path.GetTempPath();
System.IO.FileInfo dirPath = new System.IO.FileInfo(tempDir + "\\" + "lucene.test.stress");
directory = FSDirectory.GetDirectory(dirPath);
- RunStressTest(directory);
+ RunStressTest(directory, true, null);
directory.Close();
- RmDir(dirPath);
- }
-
- private void RmDir(System.IO.FileInfo dir)
- {
- System.IO.FileInfo[] files = SupportClass.FileSupport.GetFiles(dir);
- for (int i = 0; i < files.Length; i++)
- {
- bool tmpBool;
- if (System.IO.File.Exists(files[i].FullName))
- {
- System.IO.File.Delete(files[i].FullName);
- tmpBool = true;
- }
- else if (System.IO.Directory.Exists(files[i].FullName))
- {
- System.IO.Directory.Delete(files[i].FullName);
- tmpBool = true;
- }
- else
- tmpBool = false;
- bool generatedAux = tmpBool;
- }
- bool tmpBool2;
- if (System.IO.File.Exists(dir.FullName))
- {
- System.IO.File.Delete(dir.FullName);
- tmpBool2 = true;
- }
- else if (System.IO.Directory.Exists(dir.FullName))
- {
- System.IO.Directory.Delete(dir.FullName);
- tmpBool2 = true;
- }
- else
- tmpBool2 = false;
- bool generatedAux2 = tmpBool2;
+
+ // With ConcurrentMergeScheduler, in RAMDir
+ directory = new MockRAMDirectory();
+ RunStressTest(directory, true, new ConcurrentMergeScheduler());
+ directory.Close();
+
+ // With ConcurrentMergeScheduler, in FSDir
+ directory = FSDirectory.GetDirectory(dirPath);
+ RunStressTest(directory, true, new ConcurrentMergeScheduler());
+ directory.Close();
+
+ // With ConcurrentMergeScheduler and autoCommit=false, in RAMDir
+ directory = new MockRAMDirectory();
+ RunStressTest(directory, false, new ConcurrentMergeScheduler());
+ directory.Close();
+
+ // With ConcurrentMergeScheduler and autoCommit=false, in FSDir
+ directory = FSDirectory.GetDirectory(dirPath);
+ RunStressTest(directory, false, new ConcurrentMergeScheduler());
+ directory.Close();
+
+ _TestUtil.RmDir(dirPath);
}
}
}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing2.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestStressIndexing2.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing2.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestStressIndexing2.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,565 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using Lucene.Net.Documents;
+using Lucene.Net.Store;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using Lucene.Net.Analysis;
+
+using NUnit.Framework;
+
+namespace Lucene.Net.Index
+{
+
+ [TestFixture]
+ public class TestStressIndexing2 : LuceneTestCase
+ {
+ internal class AnonymousClassComparator : System.Collections.IComparer
+ {
+ public virtual int Compare(System.Object o1, System.Object o2)
+ {
+ return String.CompareOrdinal(((Fieldable) o1).Name(), ((Fieldable) o2).Name());
+ }
+ }
+ internal static int maxFields = 4;
+ internal static int bigFieldSize = 10;
+ internal static bool sameFieldOrder = false;
+ internal static bool autoCommit = false;
+ internal static int mergeFactor = 3;
+ internal static int maxBufferedDocs = 3;
+ internal static int seed = 0;
+
+ internal static System.Random r = new System.Random((System.Int32) 0);
+
+
+ [Test]
+ public virtual void TestRandom()
+ {
+ Directory dir1 = new MockRAMDirectory();
+ // dir1 = FSDirectory.getDirectory("foofoofoo");
+ Directory dir2 = new MockRAMDirectory();
+ // mergeFactor=2; maxBufferedDocs=2; Map docs = indexRandom(1, 3, 2, dir1);
+ System.Collections.IDictionary docs = IndexRandom(10, 100, 100, dir1);
+ IndexSerial(docs, dir2);
+
+ // verifying verify
+ // verifyEquals(dir1, dir1, "id");
+ // verifyEquals(dir2, dir2, "id");
+
+ VerifyEquals(dir1, dir2, "id");
+ }
+
+ [Test]
+ public virtual void TestMultiConfig()
+ {
+ // test lots of smaller different params together
+ for (int i = 0; i < 100; i++)
+ {
+ // increase iterations for better testing
+ sameFieldOrder = r.NextDouble() > 0.5;
+ autoCommit = r.NextDouble() > 0.5;
+ mergeFactor = r.Next(3) + 2;
+ maxBufferedDocs = r.Next(3) + 2;
+ seed++;
+
+ int nThreads = r.Next(5) + 1;
+ int iter = r.Next(10) + 1;
+ int range = r.Next(20) + 1;
+
+ Directory dir1 = new MockRAMDirectory();
+ Directory dir2 = new MockRAMDirectory();
+ System.Collections.IDictionary docs = IndexRandom(nThreads, iter, range, dir1);
+ IndexSerial(docs, dir2);
+ VerifyEquals(dir1, dir2, "id");
+ }
+ }
+
+
+ internal static Term idTerm = new Term("id", "");
+ internal IndexingThread[] threads;
+ internal static System.Collections.IComparer fieldNameComparator;
+
+ // This test avoids using any extra synchronization in the multiple
+ // indexing threads to test that IndexWriter does correctly synchronize
+ // everything.
+
+ public virtual System.Collections.IDictionary IndexRandom(int nThreads, int iterations, int range, Directory dir)
+ {
+ IndexWriter w = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
+ w.SetUseCompoundFile(false);
+ /***
+ w.setMaxMergeDocs(Integer.MAX_VALUE);
+ w.setMaxFieldLength(10000);
+ w.setRAMBufferSizeMB(1);
+ w.setMergeFactor(10);
+ ***/
+
+ // force many merges
+ w.SetMergeFactor(mergeFactor);
+ w.SetRAMBufferSizeMB(.1);
+ w.SetMaxBufferedDocs(maxBufferedDocs);
+
+ threads = new IndexingThread[nThreads];
+ for (int i = 0; i < threads.Length; i++)
+ {
+ IndexingThread th = new IndexingThread();
+ th.w = w;
+ th.base_Renamed = 1000000 * i;
+ th.range = range;
+ th.iterations = iterations;
+ threads[i] = th;
+ }
+
+ for (int i = 0; i < threads.Length; i++)
+ {
+ threads[i].Start();
+ }
+ for (int i = 0; i < threads.Length; i++)
+ {
+ threads[i].Join();
+ }
+
+ // w.optimize();
+ w.Close();
+
+ System.Collections.IDictionary docs = new System.Collections.Hashtable();
+ for (int i = 0; i < threads.Length; i++)
+ {
+ IndexingThread th = threads[i];
+ lock (th)
+ {
+ System.Collections.IEnumerator e = th.docs.Keys.GetEnumerator();
+ while (e.MoveNext())
+ {
+ docs[e.Current] = th.docs[e.Current];
+ }
+ }
+ }
+
+ return docs;
+ }
+
+
+ public static void IndexSerial(System.Collections.IDictionary docs, Directory dir)
+ {
+ IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer());
+
+ // index all docs in a single thread
+ System.Collections.IEnumerator iter = docs.Values.GetEnumerator();
+ while (iter.MoveNext())
+ {
+ Document d = (Document) iter.Current;
+ System.Collections.ArrayList fields = new System.Collections.ArrayList();
+ fields.AddRange(d.GetFields());
+ // put fields in same order each time
+ fields.Sort(fieldNameComparator);
+
+ Document d1 = new Document();
+ d1.SetBoost(d.GetBoost());
+ for (int i = 0; i < fields.Count; i++)
+ {
+ d1.Add((Fieldable) fields[i]);
+ }
+ w.AddDocument(d1);
+ // System.out.println("indexing "+d1);
+ }
+
+ w.Close();
+ }
+
+ public static void VerifyEquals(Directory dir1, Directory dir2, System.String idField)
+ {
+ IndexReader r1 = IndexReader.Open(dir1);
+ IndexReader r2 = IndexReader.Open(dir2);
+ VerifyEquals(r1, r2, idField);
+ r1.Close();
+ r2.Close();
+ }
+
+
+ public static void VerifyEquals(IndexReader r1, IndexReader r2, System.String idField)
+ {
+ Assert.AreEqual(r1.NumDocs(), r2.NumDocs());
+ bool hasDeletes = !(r1.MaxDoc() == r2.MaxDoc() && r1.NumDocs() == r1.MaxDoc());
+
+ int[] r2r1 = new int[r2.MaxDoc()]; // r2 id to r1 id mapping
+
+ TermDocs termDocs1 = r1.TermDocs();
+ TermDocs termDocs2 = r2.TermDocs();
+
+ // create mapping from id2 space to id2 based on idField
+ idField = String.Intern(idField);
+ TermEnum termEnum = r1.Terms(new Term(idField, ""));
+ do
+ {
+ Term term = termEnum.Term();
+ if (term == null || (System.Object) term.Field() != (System.Object) idField)
+ break;
+
+ termDocs1.Seek(termEnum);
+ Assert.IsTrue(termDocs1.Next());
+ int id1 = termDocs1.Doc();
+ Assert.IsFalse(termDocs1.Next());
+
+ termDocs2.Seek(termEnum);
+ Assert.IsTrue(termDocs2.Next());
+ int id2 = termDocs2.Doc();
+ Assert.IsFalse(termDocs2.Next());
+
+ r2r1[id2] = id1;
+
+ // verify stored fields are equivalent
+ VerifyEquals(r1.Document(id1), r2.Document(id2));
+
+ try
+ {
+ // verify term vectors are equivalent
+ VerifyEquals(r1.GetTermFreqVectors(id1), r2.GetTermFreqVectors(id2));
+ }
+ catch (System.ApplicationException e)
+ {
+ System.Console.Out.WriteLine("FAILED id=" + term + " id1=" + id1 + " id2=" + id2);
+ TermFreqVector[] tv1 = r1.GetTermFreqVectors(id1);
+ System.Console.Out.WriteLine(" d1=" + tv1);
+ if (tv1 != null)
+ for (int i = 0; i < tv1.Length; i++)
+ {
+ System.Console.Out.WriteLine(" " + i + ": " + tv1[i]);
+ }
+
+ TermFreqVector[] tv2 = r2.GetTermFreqVectors(id2);
+ System.Console.Out.WriteLine(" d2=" + tv2);
+ if (tv2 != null)
+ for (int i = 0; i < tv2.Length; i++)
+ {
+ System.Console.Out.WriteLine(" " + i + ": " + tv2[i]);
+ }
+
+ throw e;
+ }
+ }
+ while (termEnum.Next());
+
+ termEnum.Close();
+
+ // Verify postings
+ TermEnum termEnum1 = r1.Terms(new Term("", ""));
+ TermEnum termEnum2 = r2.Terms(new Term("", ""));
+
+ // pack both doc and freq into single element for easy sorting
+ long[] info1 = new long[r1.NumDocs()];
+ long[] info2 = new long[r2.NumDocs()];
+
+ for (; ; )
+ {
+ Term term1, term2;
+
+ // iterate until we get some docs
+ int len1;
+ for (; ; )
+ {
+ len1 = 0;
+ term1 = termEnum1.Term();
+ if (term1 == null)
+ break;
+ termDocs1.Seek(termEnum1);
+ while (termDocs1.Next())
+ {
+ int d1 = termDocs1.Doc();
+ int f1 = termDocs1.Freq();
+ info1[len1] = (((long) d1) << 32) | f1;
+ len1++;
+ }
+ if (len1 > 0)
+ break;
+ if (!termEnum1.Next())
+ break;
+ }
+
+ // iterate until we get some docs
+ int len2;
+ for (; ; )
+ {
+ len2 = 0;
+ term2 = termEnum2.Term();
+ if (term2 == null)
+ break;
+ termDocs2.Seek(termEnum2);
+ while (termDocs2.Next())
+ {
+ int d2 = termDocs2.Doc();
+ int f2 = termDocs2.Freq();
+ info2[len2] = (((long) r2r1[d2]) << 32) | f2;
+ len2++;
+ }
+ if (len2 > 0)
+ break;
+ if (!termEnum2.Next())
+ break;
+ }
+
+ if (!hasDeletes)
+ Assert.AreEqual(termEnum1.DocFreq(), termEnum2.DocFreq());
+
+ Assert.AreEqual(len1, len2);
+ if (len1 == 0)
+ break; // no more terms
+
+ Assert.AreEqual(term1, term2);
+
+ // sort info2 to get it into ascending docid
+ System.Array.Sort(info2, 0, len2 - 0);
+
+ // now compare
+ for (int i = 0; i < len1; i++)
+ {
+ Assert.AreEqual(info1[i], info2[i]);
+ }
+
+ termEnum1.Next();
+ termEnum2.Next();
+ }
+ }
+
+ public static void VerifyEquals(Document d1, Document d2)
+ {
+ System.Collections.ArrayList ff1 = new System.Collections.ArrayList(d1.GetFields());
+ System.Collections.ArrayList ff2 = new System.Collections.ArrayList(d2.GetFields());
+
+ ff1.Sort(fieldNameComparator);
+ ff2.Sort(fieldNameComparator);
+
+ if (ff1.Count != ff2.Count)
+ {
+ System.Console.Out.WriteLine(ff1);
+ System.Console.Out.WriteLine(ff2);
+ Assert.AreEqual(ff1.Count, ff2.Count);
+ }
+
+ for (int i = 0; i < ff1.Count; i++)
+ {
+ Fieldable f1 = (Fieldable) ff1[i];
+ Fieldable f2 = (Fieldable) ff2[i];
+ if (f1.IsBinary())
+ {
+ System.Diagnostics.Debug.Assert(f2.IsBinary());
+ //TODO
+ }
+ else
+ {
+ System.String s1 = f1.StringValue();
+ System.String s2 = f2.StringValue();
+ if (!s1.Equals(s2))
+ {
+ // print out whole doc on error
+ System.Console.Out.WriteLine(ff1);
+ System.Console.Out.WriteLine(ff2);
+ Assert.AreEqual(s1, s2);
+ }
+ }
+ }
+ }
+
+ public static void VerifyEquals(TermFreqVector[] d1, TermFreqVector[] d2)
+ {
+ if (d1 == null)
+ {
+ Assert.IsTrue(d2 == null);
+ return ;
+ }
+ Assert.IsTrue(d2 != null);
+
+ Assert.AreEqual(d1.Length, d2.Length);
+ for (int i = 0; i < d1.Length; i++)
+ {
+ TermFreqVector v1 = d1[i];
+ TermFreqVector v2 = d2[i];
+ Assert.AreEqual(v1.Size(), v2.Size());
+ int numTerms = v1.Size();
+ System.String[] terms1 = v1.GetTerms();
+ System.String[] terms2 = v2.GetTerms();
+ int[] freq1 = v1.GetTermFrequencies();
+ int[] freq2 = v2.GetTermFrequencies();
+ for (int j = 0; j < numTerms; j++)
+ {
+ if (!terms1[j].Equals(terms2[j]))
+ Assert.AreEqual(terms1[j], terms2[j]);
+ Assert.AreEqual(freq1[j], freq2[j]);
+ }
+ if (v1 is TermPositionVector)
+ {
+ Assert.IsTrue(v2 is TermPositionVector);
+ TermPositionVector tpv1 = (TermPositionVector) v1;
+ TermPositionVector tpv2 = (TermPositionVector) v2;
+ for (int j = 0; j < numTerms; j++)
+ {
+ int[] pos1 = tpv1.GetTermPositions(j);
+ int[] pos2 = tpv2.GetTermPositions(j);
+ Assert.AreEqual(pos1.Length, pos2.Length);
+ TermVectorOffsetInfo[] offsets1 = tpv1.GetOffsets(j);
+ TermVectorOffsetInfo[] offsets2 = tpv2.GetOffsets(j);
+ if (offsets1 == null)
+ Assert.IsTrue(offsets2 == null);
+ else
+ Assert.IsTrue(offsets2 != null);
+ for (int k = 0; k < pos1.Length; k++)
+ {
+ Assert.AreEqual(pos1[k], pos2[k]);
+ if (offsets1 != null)
+ {
+ Assert.AreEqual(offsets1[k].GetStartOffset(), offsets2[k].GetStartOffset());
+ Assert.AreEqual(offsets1[k].GetEndOffset(), offsets2[k].GetEndOffset());
+ }
+ }
+ }
+ }
+ }
+ }
+
+ internal class IndexingThread : SupportClass.ThreadClass
+ {
+ internal IndexWriter w;
+ internal int base_Renamed;
+ internal int range;
+ internal int iterations;
+ internal System.Collections.IDictionary docs = new System.Collections.Hashtable(); // Map<String,Document>
+ internal System.Random r;
+
+ public virtual int NextInt(int lim)
+ {
+ return r.Next(lim);
+ }
+
+ public virtual System.String GetString(int nTokens)
+ {
+ nTokens = nTokens != 0?nTokens:r.Next(4) + 1;
+ // avoid StringBuffer because it adds extra synchronization.
+ char[] arr = new char[nTokens * 2];
+ for (int i = 0; i < nTokens; i++)
+ {
+ arr[i * 2] = (char) ('A' + r.Next(10));
+ arr[i * 2 + 1] = ' ';
+ }
+ return new System.String(arr);
+ }
+
+
+ public virtual void IndexDoc()
+ {
+ Document d = new Document();
+
+ System.Collections.ArrayList fields = new System.Collections.ArrayList();
+ int id = base_Renamed + NextInt(range);
+ System.String idString = "" + id;
+ Field idField = new Field("id", idString, Field.Store.YES, Field.Index.NO_NORMS);
+ fields.Add(idField);
+
+ int nFields = NextInt(Lucene.Net.Index.TestStressIndexing2.maxFields);
+ for (int i = 0; i < nFields; i++)
+ {
+
+ Field.TermVector tvVal = Field.TermVector.NO;
+ switch (NextInt(4))
+ {
+
+ case 0:
+ tvVal = Field.TermVector.NO;
+ break;
+
+ case 1:
+ tvVal = Field.TermVector.YES;
+ break;
+
+ case 2:
+ tvVal = Field.TermVector.WITH_POSITIONS;
+ break;
+
+ case 3:
+ tvVal = Field.TermVector.WITH_POSITIONS_OFFSETS;
+ break;
+ }
+
+ switch (NextInt(4))
+ {
+
+ case 0:
+ fields.Add(new Field("f0", GetString(1), Field.Store.YES, Field.Index.NO_NORMS, tvVal));
+ break;
+
+ case 1:
+ fields.Add(new Field("f1", GetString(0), Field.Store.NO, Field.Index.TOKENIZED, tvVal));
+ break;
+
+ case 2:
+ fields.Add(new Field("f2", GetString(0), Field.Store.YES, Field.Index.NO, Field.TermVector.NO));
+ break;
+
+ case 3:
+ fields.Add(new Field("f3", GetString(Lucene.Net.Index.TestStressIndexing2.bigFieldSize), Field.Store.YES, Field.Index.TOKENIZED, tvVal));
+ break;
+ }
+ }
+
+ if (Lucene.Net.Index.TestStressIndexing2.sameFieldOrder)
+ {
+ fields.Sort(Lucene.Net.Index.TestStressIndexing2.fieldNameComparator);
+ }
+ else
+ {
+ // random placement of id field also
+ fields[NextInt(fields.Count)] = 0;
+ }
+
+ for (int i = 0; i < fields.Count; i++)
+ {
+ d.Add((Fieldable) fields[i]);
+ }
+ w.UpdateDocument(Lucene.Net.Index.TestStressIndexing2.idTerm.CreateTerm(idString), d);
+ // System.out.println("indexing "+d);
+ docs[idString] = d;
+ }
+
+ override public void Run()
+ {
+ try
+ {
+ r = new System.Random((System.Int32) (base_Renamed + range + Lucene.Net.Index.TestStressIndexing2.seed));
+ for (int i = 0; i < iterations; i++)
+ {
+ IndexDoc();
+ }
+ }
+ catch (System.Exception e)
+ {
+ System.Console.Error.WriteLine(e.StackTrace);
+ Assert.Fail(e.ToString());
+ }
+
+ lock (this)
+ {
+ int generatedAux = docs.Count;
+ }
+ }
+ }
+ static TestStressIndexing2()
+ {
+ fieldNameComparator = new AnonymousClassComparator();
+ }
+ }
+}
\ No newline at end of file