You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by do...@apache.org on 2009/07/29 20:04:24 UTC
svn commit: r798995 [28/35] - in /incubator/lucene.net/trunk/C#/src:
Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Analysis/Standard/
Lucene.Net/Document/ Lucene.Net/Index/ Lucene.Net/QueryParser/
Lucene.Net/Search/ Lucene.Net/Search/Function/ Lucene.Net...
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterDelete.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterDelete.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterDelete.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterDelete.cs Wed Jul 29 18:04:12 2009
@@ -19,24 +19,21 @@
using NUnit.Framework;
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
using Document = Lucene.Net.Documents.Document;
using Field = Lucene.Net.Documents.Field;
-using Directory = Lucene.Net.Store.Directory;
-using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
-using RAMDirectory = Lucene.Net.Store.RAMDirectory;
-using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
-using Hits = Lucene.Net.Search.Hits;
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+using ScoreDoc = Lucene.Net.Search.ScoreDoc;
using TermQuery = Lucene.Net.Search.TermQuery;
+using Directory = Lucene.Net.Store.Directory;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Index
{
-
[TestFixture]
public class TestIndexWriterDelete : LuceneTestCase
{
-
private class AnonymousClassFailure : MockRAMDirectory.Failure
{
public AnonymousClassFailure(TestIndexWriterDelete enclosingInstance)
@@ -150,7 +147,7 @@
{
bool autoCommit = (0 == pass);
- Directory dir = new RAMDirectory();
+ Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetUseCompoundFile(true);
modifier.SetMaxBufferedDeleteTerms(1);
@@ -158,39 +155,24 @@
for (int i = 0; i < keywords.Length; i++)
{
Document doc = new Document();
- doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.UN_TOKENIZED));
+ doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO));
- doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.TOKENIZED));
- doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.TOKENIZED));
+ doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED));
+ doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED));
modifier.AddDocument(doc);
}
modifier.Optimize();
-
- if (!autoCommit)
- {
- modifier.Close();
- }
+ modifier.Commit();
Term term = new Term("city", "Amsterdam");
int hitCount = GetHitCount(dir, term);
Assert.AreEqual(1, hitCount);
- if (!autoCommit)
- {
- modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
- modifier.SetUseCompoundFile(true);
- }
modifier.DeleteDocuments(term);
- if (!autoCommit)
- {
- modifier.Close();
- }
+ modifier.Commit();
hitCount = GetHitCount(dir, term);
Assert.AreEqual(0, hitCount);
- if (autoCommit)
- {
- modifier.Close();
- }
+ modifier.Close();
dir.Close();
}
}
@@ -203,7 +185,7 @@
{
bool autoCommit = (0 == pass);
- Directory dir = new RAMDirectory();
+ Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(2);
modifier.SetMaxBufferedDeleteTerms(2);
@@ -215,42 +197,25 @@
{
AddDoc(modifier, ++id, value_Renamed);
}
- modifier.Flush();
+ modifier.Commit();
Assert.AreEqual(0, modifier.GetNumBufferedDocuments());
Assert.IsTrue(0 < modifier.GetSegmentCount());
-
- if (!autoCommit)
- {
- modifier.Close();
- }
+
+ modifier.Commit();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
- if (!autoCommit)
- {
- modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
- modifier.SetMaxBufferedDocs(2);
- modifier.SetMaxBufferedDeleteTerms(2);
- }
-
- modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
-
- if (!autoCommit)
- {
- modifier.Close();
- }
+
+ modifier.Commit();
reader = IndexReader.Open(dir);
Assert.AreEqual(0, reader.NumDocs());
reader.Close();
- if (autoCommit)
- {
- modifier.Close();
- }
+ modifier.Close();
dir.Close();
}
}
@@ -261,43 +226,47 @@
{
for (int pass = 0; pass < 2; pass++)
{
- bool autoCommit = (0 == pass);
- Directory dir = new RAMDirectory();
- IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
- modifier.SetMaxBufferedDocs(4);
- modifier.SetMaxBufferedDeleteTerms(4);
-
- int id = 0;
- int value_Renamed = 100;
-
- AddDoc(modifier, ++id, value_Renamed);
- modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
- AddDoc(modifier, ++id, value_Renamed);
- modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
-
- Assert.AreEqual(2, modifier.GetNumBufferedDeleteTerms());
- Assert.AreEqual(1, modifier.GetBufferedDeleteTermsSize());
-
- AddDoc(modifier, ++id, value_Renamed);
- Assert.AreEqual(0, modifier.GetSegmentCount());
- modifier.Flush();
-
- if (!autoCommit)
- {
- modifier.Close();
- }
-
- IndexReader reader = IndexReader.Open(dir);
- Assert.AreEqual(1, reader.NumDocs());
-
- int hitCount = GetHitCount(dir, new Term("id", System.Convert.ToString(id)));
- Assert.AreEqual(1, hitCount);
- reader.Close();
- if (autoCommit)
- {
- modifier.Close();
- }
- dir.Close();
+ for (int t = 0; t < 2; t++)
+ {
+ bool autoCommit = (0 == pass);
+ Directory dir = new MockRAMDirectory();
+ IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
+ modifier.SetMaxBufferedDocs(4);
+ modifier.SetMaxBufferedDeleteTerms(4);
+
+ int id = 0;
+ int value_Renamed = 100;
+
+ AddDoc(modifier, ++id, value_Renamed);
+ if (0 == t)
+ modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
+ else
+ modifier.DeleteDocuments(new TermQuery(new Term("value", System.Convert.ToString(value_Renamed))));
+ AddDoc(modifier, ++id, value_Renamed);
+ if (0 == t)
+ {
+ modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
+ Assert.AreEqual(2, modifier.GetNumBufferedDeleteTerms());
+ Assert.AreEqual(1, modifier.GetBufferedDeleteTermsSize());
+ }
+ else
+ modifier.DeleteDocuments(new TermQuery(new Term("value", System.Convert.ToString(value_Renamed))));
+
+ AddDoc(modifier, ++id, value_Renamed);
+ Assert.AreEqual(0, modifier.GetSegmentCount());
+ modifier.Flush();
+
+ modifier.Commit();
+
+ IndexReader reader = IndexReader.Open(dir);
+ Assert.AreEqual(1, reader.NumDocs());
+
+ int hitCount = GetHitCount(dir, new Term("id", System.Convert.ToString(id)));
+ Assert.AreEqual(1, hitCount);
+ reader.Close();
+ modifier.Close();
+ dir.Close();
+ }
}
}
@@ -309,7 +278,7 @@
{
bool autoCommit = (0 == pass);
- Directory dir = new RAMDirectory();
+ Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(100);
modifier.SetMaxBufferedDeleteTerms(100);
@@ -327,26 +296,19 @@
{
AddDoc(modifier, ++id, value_Renamed);
}
- modifier.Flush();
+ modifier.Commit();
for (int i = 0; i < 5; i++)
{
AddDoc(modifier, ++id, value_Renamed);
}
modifier.DeleteDocuments(new Term("value", System.Convert.ToString(value_Renamed)));
-
- modifier.Flush();
- if (!autoCommit)
- {
- modifier.Close();
- }
+
+ modifier.Commit();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(5, reader.NumDocs());
- if (autoCommit)
- {
- modifier.Close();
- }
+ modifier.Close();
}
}
@@ -357,7 +319,7 @@
for (int pass = 0; pass < 2; pass++)
{
bool autoCommit = (0 == pass);
- Directory dir = new RAMDirectory();
+ Directory dir = new MockRAMDirectory();
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
modifier.SetMaxBufferedDocs(2);
modifier.SetMaxBufferedDeleteTerms(2);
@@ -369,31 +331,17 @@
{
AddDoc(modifier, ++id, value_Renamed);
}
- modifier.Flush();
- if (!autoCommit)
- {
- modifier.Close();
- }
+ modifier.Commit();
IndexReader reader = IndexReader.Open(dir);
Assert.AreEqual(7, reader.NumDocs());
reader.Close();
- if (!autoCommit)
- {
- modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
- modifier.SetMaxBufferedDocs(2);
- modifier.SetMaxBufferedDeleteTerms(2);
- }
-
id = 0;
modifier.DeleteDocuments(new Term("id", System.Convert.ToString(++id)));
modifier.DeleteDocuments(new Term("id", System.Convert.ToString(++id)));
-
- if (!autoCommit)
- {
- modifier.Close();
- }
+
+ modifier.Commit();
reader = IndexReader.Open(dir);
Assert.AreEqual(5, reader.NumDocs());
@@ -404,25 +352,13 @@
{
terms[i] = new Term("id", System.Convert.ToString(++id));
}
- if (!autoCommit)
- {
- modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
- modifier.SetMaxBufferedDocs(2);
- modifier.SetMaxBufferedDeleteTerms(2);
- }
modifier.DeleteDocuments(terms);
- if (!autoCommit)
- {
- modifier.Close();
- }
- reader = IndexReader.Open(dir);
+ modifier.Commit();
+ reader = IndexReader.Open(dir);
Assert.AreEqual(2, reader.NumDocs());
reader.Close();
- if (autoCommit)
- {
- modifier.Close();
- }
+ modifier.Close();
dir.Close();
}
}
@@ -430,16 +366,16 @@
private void AddDoc(IndexWriter modifier, int id, int value_Renamed)
{
Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
- doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.TOKENIZED));
- doc.Add(new Field("id", System.Convert.ToString(id), Field.Store.YES, Field.Index.UN_TOKENIZED));
- doc.Add(new Field("value", System.Convert.ToString(value_Renamed), Field.Store.NO, Field.Index.UN_TOKENIZED));
+ doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
+ doc.Add(new Field("id", System.Convert.ToString(id), Field.Store.YES, Field.Index.NOT_ANALYZED));
+ doc.Add(new Field("value", System.Convert.ToString(value_Renamed), Field.Store.NO, Field.Index.NOT_ANALYZED));
modifier.AddDocument(doc);
}
private int GetHitCount(Directory dir, Term term)
{
IndexSearcher searcher = new IndexSearcher(dir);
- int hitCount = searcher.Search(new TermQuery(term)).Length();
+ int hitCount = searcher.Search(new TermQuery(term), null, 1000).totalHits;
searcher.Close();
return hitCount;
}
@@ -472,13 +408,13 @@
bool autoCommit = (0 == pass);
// First build up a starting index:
- RAMDirectory startDir = new RAMDirectory();
+ MockRAMDirectory startDir = new MockRAMDirectory();
IndexWriter writer = new IndexWriter(startDir, autoCommit, new WhitespaceAnalyzer(), true);
for (int i = 0; i < 157; i++)
{
Document d = new Document();
- d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.UN_TOKENIZED));
- d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
+ d.Add(new Field("content", "aaa " + i, Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(d);
}
writer.Close();
@@ -494,6 +430,7 @@
while (!done)
{
MockRAMDirectory dir = new MockRAMDirectory(startDir);
+ dir.SetPreventDoubleWrite(false);
IndexWriter modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
modifier.SetMaxBufferedDocs(1000); // use flush or close
@@ -559,8 +496,8 @@
if (updates)
{
Document d = new Document();
- d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.UN_TOKENIZED));
- d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("id", System.Convert.ToString(i), Field.Store.YES, Field.Index.NOT_ANALYZED));
+ d.Add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED));
modifier.UpdateDocument(new Term("id", System.Convert.ToString(docId)), d);
}
else
@@ -583,7 +520,6 @@
{
if (debug)
{
- //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
System.Console.Out.WriteLine(" hit IOException: " + e);
System.Console.Out.WriteLine(e.StackTrace);
}
@@ -595,43 +531,12 @@
}
}
- // Whether we succeeded or failed, check that all
- // un-referenced files were in fact deleted (ie,
- // we did not create garbage). Just create a
- // new IndexFileDeleter, have it delete
- // unreferenced files, then verify that in fact
- // no files were deleted:
- System.String[] startFiles = dir.List();
- SegmentInfos infos = new SegmentInfos();
- infos.Read(dir);
- new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
- System.String[] endFiles = dir.List();
-
- //UPGRADE_TODO: Method 'java.util.Arrays.sort' was converted to 'System.Array.Sort' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javautilArrayssort_javalangObject[]'"
- System.Array.Sort(startFiles);
- //UPGRADE_TODO: Method 'java.util.Arrays.sort' was converted to 'System.Array.Sort' which has a different behavior. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1073_javautilArrayssort_javalangObject[]'"
- System.Array.Sort(endFiles);
-
- // for(int i=0;i<startFiles.length;i++) {
- // System.out.println(" startFiles: " + i + ": " + startFiles[i]);
- // }
-
- if (!SupportClass.Compare.CompareStringArrays(startFiles, endFiles))
- {
- System.String successStr;
- if (success)
- {
- successStr = "success";
- }
- else
- {
- successStr = "IOException";
- System.Console.Error.WriteLine(err.StackTrace);
- }
- Assert.Fail("reader.close() failed to delete unreferenced files after " + successStr + " (" + diskFree + " bytes): before delete:\n " + ArrayToString(startFiles) + "\n after delete:\n " + ArrayToString(endFiles));
- }
-
- // Finally, verify index is not corrupt, and, if
+
+ // If the Close() succeeded, make sure there are no unreferenced files
+ if (success)
+ TestIndexWriter.AssertNoUnreferencedFiles(dir, "after writer.Close");
+
+ // Finally, verify index is not corrupt, and, if
// we succeeded, we see all docs changed, and if
// we failed, we see either all docs or no docs
// changed (transactional semantics):
@@ -643,23 +548,21 @@
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
- //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
Assert.Fail(testName + ":exception when creating IndexReader after disk full during close: " + e);
}
IndexSearcher searcher = new IndexSearcher(newReader);
- Hits hits = null;
+ ScoreDoc[] hits = null;
try
{
- hits = searcher.Search(new TermQuery(searchTerm));
+ hits = searcher.Search(new TermQuery(searchTerm), null, 1000).scoreDocs;
}
catch (System.IO.IOException e)
{
System.Console.Error.WriteLine(e.StackTrace);
- //UPGRADE_TODO: The equivalent in .NET for method 'java.lang.Throwable.toString' may return a different value. "ms-help://MS.VSCC.v80/dv_commoner/local/redirect.htm?index='!DefaultContextWindowIndex'&keyword='jlca1043'"
Assert.Fail(testName + ": exception when searching: " + e);
}
- int result2 = hits.Length();
+ int result2 = hits.Length;
if (success)
{
if (x == 0 && result2 != END_COUNT)
@@ -733,23 +636,18 @@
for (int i = 0; i < keywords.Length; i++)
{
Document doc = new Document();
- doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.UN_TOKENIZED));
+ doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO));
- doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.TOKENIZED));
- doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.TOKENIZED));
+ doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED));
+ doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED));
modifier.AddDocument(doc);
}
// flush (and commit if ac)
-
- modifier.Optimize();
-
- // commit if !ac
-
- if (!autoCommit)
- {
- modifier.Close();
- }
- // one of the two files hits
+
+ modifier.Optimize();
+ modifier.Commit();
+
+ // one of the two files hits
Term term = new Term("city", "Amsterdam");
int hitCount = GetHitCount(dir, term);
@@ -757,12 +655,6 @@
// open the writer again (closed above)
- if (!autoCommit)
- {
- modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
- modifier.SetUseCompoundFile(true);
- }
-
// delete the doc
// max buf del terms is two, so this is buffered
@@ -774,7 +666,7 @@
Document doc2 = new Document();
modifier.AddDocument(doc2);
- // flush the changes, the buffered deletes, and the new doc
+ // commit the changes, the buffered deletes, and the new doc
// The failure object will fail on the first write after the del
// file gets created when processing the buffered delete
@@ -790,7 +682,7 @@
bool failed = false;
try
{
- modifier.Flush();
+ modifier.Commit();
}
catch (System.IO.IOException)
{
@@ -799,32 +691,18 @@
Assert.IsTrue(failed);
- // The flush above failed, so we need to retry it (which will
+ // The commit above failed, so we need to retry it (which will
// succeed, because the failure is a one-shot)
-
- if (!autoCommit)
- {
- modifier.Close();
- }
- else
- {
- modifier.Flush();
- }
+
+ modifier.Commit();
hitCount = GetHitCount(dir, term);
- // If the delete was not cleared then hit count will
- // be 0. With autoCommit=false, we hit the exception
- // on creating the compound file, so the delete was
- // flushed successfully.
- Assert.AreEqual(autoCommit?1:0, hitCount);
-
- if (autoCommit)
- {
- modifier.Close();
- }
+ // Make sure the delete was successfully flushed
+ Assert.AreEqual(0, hitCount);
- dir.Close();
+ modifier.Close();
+ dir.Close();
}
}
@@ -855,10 +733,10 @@
for (int i = 0; i < keywords.Length; i++)
{
Document doc = new Document();
- doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.UN_TOKENIZED));
+ doc.Add(new Field("id", keywords[i], Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field("country", unindexed[i], Field.Store.YES, Field.Index.NO));
- doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.TOKENIZED));
- doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.TOKENIZED));
+ doc.Add(new Field("contents", unstored[i], Field.Store.NO, Field.Index.ANALYZED));
+ doc.Add(new Field("city", text[i], Field.Store.YES, Field.Index.ANALYZED));
try
{
modifier.AddDocument(doc);
Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterExceptions.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterExceptions.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterExceptions.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterExceptions.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using NUnit.Framework;
+
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using _TestUtil = Lucene.Net.Util._TestUtil;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
+using Directory = Lucene.Net.Store.Directory;
+using Analyzer = Lucene.Net.Analysis.Analyzer;
+using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using CloseableThreadLocal = Lucene.Net.Util.CloseableThreadLocal;
+
+namespace Lucene.Net.Index
+{
+ [TestFixture]
+ public class TestIndexWriterExceptions : LuceneTestCase
+ {
+
+ private const bool DEBUG = false;
+
+ private class IndexerThread : SupportClass.ThreadClass
+ {
+ private TestIndexWriterExceptions enclosingInstance;
+ internal IndexWriter writer;
+
+ internal readonly System.Random r = new System.Random(47);
+ internal System.Exception failure;
+
+ public IndexerThread(int i, IndexWriter writer, TestIndexWriterExceptions enclosingInstance)
+ : base("Indexer " + i)
+ {
+ this.writer = writer;
+ this.enclosingInstance = enclosingInstance;
+ }
+
+ override public void Run()
+ {
+
+ Document doc = new Document();
+
+ doc.Add(new Field("content1", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.ANALYZED));
+ doc.Add(new Field("content6", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ doc.Add(new Field("content2", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NOT_ANALYZED));
+ doc.Add(new Field("content3", "aaa bbb ccc ddd", Field.Store.YES, Field.Index.NO));
+
+ doc.Add(new Field("content4", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.ANALYZED));
+ doc.Add(new Field("content5", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED));
+
+ doc.Add(new Field("content7", "aaa bbb ccc ddd", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+
+ Field idField = new Field("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
+ doc.Add(idField);
+
+ System.DateTime stopTime = System.DateTime.Now.AddSeconds(3);
+
+ while (System.DateTime.Now < stopTime)
+ {
+ enclosingInstance.doFail.Set(this);
+ string id = "" + r.Next(50);
+ idField.SetValue(id);
+ Term idTerm = new Term("id", id);
+ try
+ {
+ writer.UpdateDocument(idTerm, doc);
+ }
+ catch (System.Exception re)
+ {
+ if (DEBUG)
+ {
+ System.Console.Out.WriteLine("EXC: ");
+ System.Console.Out.WriteLine(re.StackTrace);
+ }
+ try
+ {
+ _TestUtil.CheckIndex(writer.GetDirectory());
+ }
+ catch (System.IO.IOException ioe)
+ {
+ System.Console.Out.WriteLine(System.Threading.Thread.CurrentThread.Name + ": unexpected exception1");
+ System.Console.Out.WriteLine(ioe.StackTrace);
+ failure = ioe;
+ break;
+ }
+ // this, in Java, was catch Throwable, and the catch above (at the same nesting level)
+ // was catch RuntimeException... as all exceptions in C# are unchecked, these both come
+ // down to System.Exception
+ /*
+ } catch (System.Exception t) {
+ System.Console.Out.WriteLine(System.Threading.Thread.CurrentThread.Name + ": unexpected exception2");
+ System.Console.Out.WriteLine(t.StackTrace);
+ failure = t;
+ break;
+ */
+ }
+
+ enclosingInstance.doFail.Set(null);
+
+ // After a possible exception (above) I should be able
+ // to add a new document without hitting an
+ // exception:
+ try
+ {
+ writer.UpdateDocument(idTerm, doc);
+ }
+ catch (System.Exception t)
+ {
+ System.Console.Out.WriteLine(System.Threading.Thread.CurrentThread.Name + ": unexpected exception3");
+ System.Console.Out.WriteLine(t.StackTrace);
+ failure = t;
+ break;
+ }
+ }
+ }
+ }
+
+ CloseableThreadLocal doFail = new CloseableThreadLocal();
+
+ public class MockIndexWriter : IndexWriter
+ {
+ private TestIndexWriterExceptions enclosingInstance;
+
+ internal System.Random r = new System.Random(17);
+
+ public MockIndexWriter(Directory dir, Analyzer a, bool create, MaxFieldLength mfl, TestIndexWriterExceptions enclosingInstance)
+ : base(dir, a, create, mfl)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+
+ protected override bool TestPoint(string name)
+ {
+ if (enclosingInstance.doFail.Get() != null && !name.Equals("startDoFlush") && r.Next(20) == 17)
+ {
+ if (DEBUG)
+ {
+ System.Console.Out.WriteLine(System.Threading.Thread.CurrentThread.Name + ": NOW FAIL: " + name);
+ }
+ throw new System.Exception(System.Threading.Thread.CurrentThread.Name + ": intentionally failing at " + name);
+ }
+ return true;
+ }
+ }
+
+ [Test]
+ public void TestRandomExceptions()
+ {
+ MockRAMDirectory dir = new MockRAMDirectory();
+
+ MockIndexWriter writer = new MockIndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, this);
+ ((ConcurrentMergeScheduler)writer.GetMergeScheduler()).SetSuppressExceptions_ForNUnitTest();
+ //writer.setMaxBufferedDocs(10);
+ writer.SetRAMBufferSizeMB(0.1);
+
+ if (DEBUG)
+ writer.SetInfoStream(System.Console.Out);
+
+ IndexerThread thread = new IndexerThread(0, writer, this);
+ thread.Run();
+ if (thread.failure != null)
+ {
+ System.Console.Out.WriteLine(thread.failure.StackTrace);
+ Assert.Fail("thread " + thread.Name + ": hit unexpected failure");
+ }
+
+ writer.Commit();
+
+ try
+ {
+ writer.Close();
+ }
+ catch (System.Exception t)
+ {
+ System.Console.Out.WriteLine("exception during close:");
+ System.Console.Out.WriteLine(t.StackTrace);
+ writer.Rollback();
+ }
+
+ // Confirm that when doc hits exception partway through tokenization, it's deleted:
+ IndexReader r2 = IndexReader.Open(dir);
+ int count = r2.DocFreq(new Term("content4", "aaa"));
+ int count2 = r2.DocFreq(new Term("content4", "ddd"));
+ Assert.AreEqual(count, count2);
+ r2.Close();
+
+ _TestUtil.CheckIndex(dir);
+ }
+
+ [Test]
+ public void TestRandomExceptionsThreads()
+ {
+
+ MockRAMDirectory dir = new MockRAMDirectory();
+ MockIndexWriter writer = new MockIndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED, this);
+ ((ConcurrentMergeScheduler)writer.GetMergeScheduler()).SetSuppressExceptions_ForNUnitTest();
+ //writer.setMaxBufferedDocs(10);
+ writer.SetRAMBufferSizeMB(0.2);
+
+ if (DEBUG)
+ writer.SetInfoStream(System.Console.Out);
+
+ int NUM_THREADS = 4;
+
+ IndexerThread[] threads = new IndexerThread[NUM_THREADS];
+ for (int i = 0; i < NUM_THREADS; i++)
+ {
+ threads[i] = new IndexerThread(i, writer, this);
+ threads[i].Start();
+ }
+
+ for (int i = 0; i < NUM_THREADS; i++)
+ threads[i].Join();
+
+ for (int i = 0; i < NUM_THREADS; i++)
+ if (threads[i].failure != null)
+ Assert.Fail("thread " + threads[i].Name + ": hit unexpected failure");
+
+ writer.Commit();
+
+ try
+ {
+ writer.Close();
+ }
+ catch (System.Exception t)
+ {
+ System.Console.Out.WriteLine("exception during close:");
+ System.Console.Out.WriteLine(t.StackTrace);
+ writer.Rollback();
+ }
+
+ // Confirm that when doc hits exception partway through tokenization, it's deleted:
+ IndexReader r2 = IndexReader.Open(dir);
+ int count = r2.DocFreq(new Term("content4", "aaa"));
+ int count2 = r2.DocFreq(new Term("content4", "ddd"));
+ Assert.AreEqual(count, count2);
+ r2.Close();
+
+ _TestUtil.CheckIndex(dir);
+ }
+ }
+}
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterLockRelease.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterLockRelease.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterLockRelease.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterLockRelease.cs Wed Jul 29 18:04:12 2009
@@ -28,8 +28,6 @@
/// exist).
///
/// </summary>
- /// <author> mbogosian
- /// </author>
/// <version> $Id$
/// </version>
@@ -130,13 +128,13 @@
try
{
- im = new IndexWriter(this.__test_dir, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), false);
+ im = new IndexWriter(this.__test_dir, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
}
catch (System.IO.FileNotFoundException)
{
try
{
- im = new IndexWriter(this.__test_dir, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), false);
+ im = new IndexWriter(this.__test_dir, new Lucene.Net.Analysis.Standard.StandardAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
}
catch (System.IO.FileNotFoundException)
{
@@ -144,4 +142,4 @@
}
}
}
-}
+}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMergePolicy.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterMergePolicy.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMergePolicy.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMergePolicy.cs Wed Jul 29 18:04:12 2009
@@ -39,8 +39,8 @@
public virtual void TestNormalCase()
{
Directory dir = new RAMDirectory();
-
- IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
writer.SetMergeFactor(10);
writer.SetMergePolicy(new LogDocMergePolicy());
@@ -59,8 +59,8 @@
public virtual void TestNoOverMerge()
{
Directory dir = new RAMDirectory();
-
- IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
writer.SetMergeFactor(10);
writer.SetMergePolicy(new LogDocMergePolicy());
@@ -85,8 +85,8 @@
public virtual void TestForceFlush()
{
Directory dir = new RAMDirectory();
-
- IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
writer.SetMergeFactor(10);
LogDocMergePolicy mp = new LogDocMergePolicy();
@@ -97,8 +97,8 @@
{
AddDoc(writer);
writer.Close();
-
- writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+
+ writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
writer.SetMergeFactor(10);
mp.SetMinMergeDocs(100);
@@ -114,8 +114,8 @@
public virtual void TestMergeFactorChange()
{
Directory dir = new RAMDirectory();
-
- IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
writer.SetMergeFactor(100);
writer.SetMergePolicy(new LogDocMergePolicy());
@@ -144,8 +144,8 @@
public virtual void TestMaxBufferedDocsChange()
{
Directory dir = new RAMDirectory();
-
- IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+
+ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(101);
writer.SetMergeFactor(101);
writer.SetMergePolicy(new LogDocMergePolicy());
@@ -160,8 +160,8 @@
CheckInvariants(writer);
}
writer.Close();
-
- writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+
+ writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(101);
writer.SetMergeFactor(101);
writer.SetMergePolicy(new LogDocMergePolicy());
@@ -193,7 +193,7 @@
{
Directory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
+ IndexWriter writer = new IndexWriter(dir, true, new WhitespaceAnalyzer(), true);
writer.SetMergePolicy(new LogDocMergePolicy());
writer.SetMaxBufferedDocs(10);
writer.SetMergeFactor(100);
@@ -209,7 +209,7 @@
reader.DeleteDocuments(new Term("content", "aaa"));
reader.Close();
- writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+ writer = new IndexWriter(dir, true, new WhitespaceAnalyzer(), false);
writer.SetMergePolicy(new LogDocMergePolicy());
writer.SetMaxBufferedDocs(10);
writer.SetMergeFactor(5);
@@ -228,7 +228,7 @@
private void AddDoc(IndexWriter writer)
{
Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
- doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.TOKENIZED));
+ doc.Add(new Field("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(doc);
}
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMerging.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestIndexWriterMerging.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMerging.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestIndexWriterMerging.cs Wed Jul 29 18:04:12 2009
@@ -62,7 +62,7 @@
Directory merged = new MockRAMDirectory();
- IndexWriter writer = new IndexWriter(merged, new StandardAnalyzer(), true);
+ IndexWriter writer = new IndexWriter(merged, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMergeFactor(2);
writer.AddIndexes(new Directory[]{indexA, indexB});
@@ -97,15 +97,15 @@
private void FillIndex(Directory dir, int start, int numDocs)
{
-
- IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true);
+
+ IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMergeFactor(2);
writer.SetMaxBufferedDocs(2);
for (int i = start; i < (start + numDocs); i++)
{
Lucene.Net.Documents.Document temp = new Lucene.Net.Documents.Document();
- temp.Add(new Field("count", ("" + i), Field.Store.YES, Field.Index.UN_TOKENIZED));
+ temp.Add(new Field("count", ("" + i), Field.Store.YES, Field.Index.NOT_ANALYZED));
writer.AddDocument(temp);
}
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyBug.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestLazyBug.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyBug.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyBug.cs Wed Jul 29 18:04:12 2009
@@ -69,7 +69,7 @@
{
System.Random r = new System.Random((System.Int32) (BASE_SEED + 42));
Analyzer analyzer = new SimpleAnalyzer();
- IndexWriter writer = new IndexWriter(dir, analyzer, true);
+ IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetUseCompoundFile(false);
@@ -78,7 +78,7 @@
Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
for (int f = 1; f <= NUM_FIELDS; f++)
{
- doc.Add(new Field("f" + f, data[f % data.Length] + '#' + data[r.Next(data.Length)], Field.Store.YES, Field.Index.TOKENIZED));
+ doc.Add(new Field("f" + f, data[f % data.Length] + '#' + data[r.Next(data.Length)], Field.Store.YES, Field.Index.ANALYZED));
}
writer.AddDocument(doc);
}
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyProxSkipping.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestLazyProxSkipping.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyProxSkipping.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestLazyProxSkipping.cs Wed Jul 29 18:04:12 2009
@@ -14,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
using System;
using NUnit.Framework;
@@ -25,7 +24,7 @@
using IndexInput = Lucene.Net.Store.IndexInput;
using RAMDirectory = Lucene.Net.Store.RAMDirectory;
using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
-using Hits = Lucene.Net.Search.Hits;
+using ScoreDoc = Lucene.Net.Search.ScoreDoc;
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
using PhraseQuery = Lucene.Net.Search.PhraseQuery;
using Searcher = Lucene.Net.Search.Searcher;
@@ -53,7 +52,7 @@
int numDocs = 500;
Directory directory = new RAMDirectory();
- IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true);
+ IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
writer.SetMaxBufferedDocs(10);
for (int i = 0; i < numDocs; i++)
{
@@ -75,7 +74,7 @@
content = this.term3 + " " + this.term2;
}
- doc.Add(new Field(this.field, content, Field.Store.YES, Field.Index.TOKENIZED));
+ doc.Add(new Field(this.field, content, Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc);
}
@@ -92,22 +91,22 @@
this.searcher = new IndexSearcher(reader);
}
- private Hits Search()
+ private ScoreDoc[] Search()
{
// create PhraseQuery "term1 term2" and search
PhraseQuery pq = new PhraseQuery();
pq.Add(new Term(this.field, this.term1));
pq.Add(new Term(this.field, this.term2));
- return this.searcher.Search(pq);
+ return this.searcher.Search(pq, null, 1000).scoreDocs;
}
private void PerformTest(int numHits)
{
CreateIndex(numHits);
this.seeksCounter = 0;
- Hits hits = Search();
+ ScoreDoc[] hits = Search();
// verify that the right number of docs was found
- Assert.AreEqual(numHits, hits.Length());
+ Assert.AreEqual(numHits, hits.Length);
// check if the number of calls of seek() does not exceed the number of hits
Assert.IsTrue(this.seeksCounter <= numHits + 1);
@@ -125,11 +124,11 @@
public virtual void TestSeek()
{
Directory directory = new RAMDirectory();
- IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true);
+ IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
for (int i = 0; i < 10; i++)
{
Document doc = new Document();
- doc.Add(new Field(this.field, "a b", Field.Store.YES, Field.Index.TOKENIZED));
+ doc.Add(new Field(this.field, "a b", Field.Store.YES, Field.Index.ANALYZED));
writer.AddDocument(doc);
}
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiLevelSkipList.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestMultiLevelSkipList.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiLevelSkipList.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiLevelSkipList.cs Wed Jul 29 18:04:12 2009
@@ -47,12 +47,12 @@
public virtual void TestSimpleSkip()
{
RAMDirectory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new PayloadAnalyzer(), true);
+ IndexWriter writer = new IndexWriter(dir, new PayloadAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
Term term = new Term("test", "a");
for (int i = 0; i < 5000; i++)
{
Document d1 = new Document();
- d1.Add(new Field(term.Field(), term.Text(), Field.Store.NO, Field.Index.TOKENIZED));
+ d1.Add(new Field(term.Field(), term.Text(), Field.Store.NO, Field.Index.ANALYZED));
writer.AddDocument(d1);
}
writer.Flush();
@@ -110,14 +110,15 @@
{
}
- public override Token Next()
+ public override Token Next(Token reusableToken)
{
- Token t = input.Next();
- if (t != null)
+ System.Diagnostics.Debug.Assert(reusableToken != null);
+ Token nextToken = input.Next(reusableToken);
+ if (nextToken != null)
{
- t.SetPayload(new Payload(new byte[]{(byte) count++}));
+ nextToken.SetPayload(new Payload(new byte[]{(byte) count++}));
}
- return t;
+ return nextToken;
}
}
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiSegmentReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestMultiSegmentReader.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiSegmentReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestMultiSegmentReader.cs Wed Jul 29 18:04:12 2009
@@ -52,8 +52,8 @@
doc2 = new Document();
DocHelper.SetupDoc(doc1);
DocHelper.SetupDoc(doc2);
- SegmentInfo info1 = DocHelper.WriteDoc(dir, doc1);
- SegmentInfo info2 = DocHelper.WriteDoc(dir, doc2);
+ DocHelper.WriteDoc(dir, doc1);
+ DocHelper.WriteDoc(dir, doc2);
sis = new SegmentInfos();
sis.Read(dir);
}
@@ -113,7 +113,7 @@
if (reader is MultiReader)
// MultiReader does not "own" the directory so it does
// not write the changes to sis on commit:
- sis.Write(dir);
+ sis.Commit(dir);
sis.Read(dir);
reader = OpenReader();
@@ -126,7 +126,7 @@
if (reader is MultiReader)
// MultiReader does not "own" the directory so it does
// not write the changes to sis on commit:
- sis.Write(dir);
+ sis.Commit(dir);
sis.Read(dir);
reader = OpenReader();
Assert.AreEqual(1, reader.NumDocs());
@@ -168,9 +168,9 @@
private void AddDoc(RAMDirectory ramDir1, System.String s, bool create)
{
- IndexWriter iw = new IndexWriter(ramDir1, new StandardAnalyzer(), create);
+ IndexWriter iw = new IndexWriter(ramDir1, new StandardAnalyzer(), create, IndexWriter.MaxFieldLength.LIMITED);
Document doc = new Document();
- doc.Add(new Field("body", s, Field.Store.YES, Field.Index.TOKENIZED));
+ doc.Add(new Field("body", s, Field.Store.YES, Field.Index.ANALYZED));
iw.AddDocument(doc);
iw.Close();
}
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestNorms.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestNorms.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestNorms.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestNorms.cs Wed Jul 29 18:04:12 2009
@@ -92,7 +92,7 @@
/// Including optimize.
/// </summary>
[Test]
- public virtual void _TestNorms()
+ public virtual void TestNorms_Renamed()
{
// tmp dir
System.String tempDir = System.IO.Path.GetTempPath();
@@ -131,7 +131,7 @@
Directory dir3 = FSDirectory.GetDirectory(indexDir3);
CreateIndex(dir3);
- IndexWriter iw = new IndexWriter(dir3, anlzr, false);
+ IndexWriter iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.AddIndexes(new Directory[]{dir1, dir2});
@@ -148,7 +148,7 @@
DoTestNorms(dir3);
// now with optimize
- iw = new IndexWriter(dir3, anlzr, false);
+ iw = new IndexWriter(dir3, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.Optimize();
@@ -177,7 +177,7 @@
private void CreateIndex(Directory dir)
{
- IndexWriter iw = new IndexWriter(dir, anlzr, true);
+ IndexWriter iw = new IndexWriter(dir, anlzr, true, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.SetSimilarity(similarityOne);
@@ -226,7 +226,7 @@
private void AddDocs(Directory dir, int ndocs, bool compound)
{
- IndexWriter iw = new IndexWriter(dir, anlzr, false);
+ IndexWriter iw = new IndexWriter(dir, anlzr, false, IndexWriter.MaxFieldLength.LIMITED);
iw.SetMaxBufferedDocs(5);
iw.SetMergeFactor(3);
iw.SetSimilarity(similarityOne);
@@ -245,7 +245,7 @@
float boost = NextNorm();
for (int i = 0; i < 10; i++)
{
- Field f = new Field("f" + i, "v" + i, Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.UN_TOKENIZED);
+ Field f = new Field("f" + i, "v" + i, Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.NOT_ANALYZED);
f.SetBoost(boost);
d.Add(f);
}
Added: incubator/lucene.net/trunk/C#/src/Test/Index/TestOmitTf.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestOmitTf.cs?rev=798995&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestOmitTf.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestOmitTf.cs Wed Jul 29 18:04:12 2009
@@ -0,0 +1,373 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using NUnit.Framework;
+
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using _TestUtil = Lucene.Net.Util._TestUtil;
+using Analyzer = Lucene.Net.Analysis.Analyzer;
+using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using BooleanQuery = Lucene.Net.Search.BooleanQuery;
+using HitCollector = Lucene.Net.Search.HitCollector;
+using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+using Searcher = Lucene.Net.Search.Searcher;
+using Similarity = Lucene.Net.Search.Similarity;
+using TermQuery = Lucene.Net.Search.TermQuery;
+using Occur = Lucene.Net.Search.BooleanClause.Occur;
+using Directory = Lucene.Net.Store.Directory;
+using MockRAMDirectory = Lucene.Net.Store.MockRAMDirectory;
+
+namespace Lucene.Net.Index
+{
+ [TestFixture]
+ public class TestOmitTf : LuceneTestCase
+ {
+
+ public class SimpleSimilarity : Similarity
+ {
+ override public float LengthNorm(string field, int numTerms) { return 1.0f; }
+ override public float QueryNorm(float sumOfSquaredWeights) { return 1.0f; }
+ override public float Tf(float freq) { return freq; }
+ override public float SloppyFreq(int distance) { return 2.0f; }
+ override public float Idf(System.Collections.ICollection terms, Searcher searcher) { return 1.0f; }
+ override public float Idf(int docFreq, int numDocs) { return 1.0f; }
+ override public float Coord(int overlap, int maxOverlap) { return 1.0f; }
+ }
+
+ // Tests whether the DocumentWriter correctly enable the
+ // omitTf bit in the FieldInfo
+ [Test]
+ public void TestOmitTf_Renamed()
+ {
+ Directory ram = new MockRAMDirectory();
+ Analyzer analyzer = new StandardAnalyzer();
+ IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+ Document d = new Document();
+
+ // this field will have Tf
+ Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
+ d.Add(f1);
+
+ // this field will NOT have Tf
+ Field f2 = new Field("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
+ f2.SetOmitTf(true);
+ d.Add(f2);
+
+ writer.AddDocument(d);
+ writer.Optimize();
+ // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger
+ // keep things constant
+ d = new Document();
+
+ // Reverese
+ f1.SetOmitTf(true);
+ d.Add(f1);
+
+ f2.SetOmitTf(false);
+ d.Add(f2);
+
+ writer.AddDocument(d);
+ // force merge
+ writer.Optimize();
+ // flush
+ writer.Close();
+ _TestUtil.CheckIndex(ram);
+
+ // only one segment in the index, so we can cast to SegmentReader
+ SegmentReader reader = (SegmentReader)IndexReader.Open(ram);
+ FieldInfos fi = reader.FieldInfos();
+ Assert.IsTrue(fi.FieldInfo("f1").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+ Assert.IsTrue(fi.FieldInfo("f2").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+
+ reader.Close();
+ ram.Close();
+ }
+
+ // Tests whether merging of docs that have different
+ // omitTf for the same field works
+ [Test]
+ public void TestMixedMerge()
+ {
+ Directory ram = new MockRAMDirectory();
+ Analyzer analyzer = new StandardAnalyzer();
+ IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+ writer.SetMaxBufferedDocs(3);
+ writer.SetMergeFactor(2);
+ Document d = new Document();
+
+ // this field will have Tf
+ Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
+ d.Add(f1);
+
+ // this field will NOT have Tf
+ Field f2 = new Field("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
+ f2.SetOmitTf(true);
+ d.Add(f2);
+
+ for (int i = 0; i < 30; i++)
+ writer.AddDocument(d);
+
+ // now we add another document which has term freq for field f2 and not for f1 and verify if the SegmentMerger
+ // keep things constant
+ d = new Document();
+
+ // Reverese
+ f1.SetOmitTf(true);
+ d.Add(f1);
+
+ f2.SetOmitTf(false);
+ d.Add(f2);
+
+ for (int i = 0; i < 30; i++)
+ writer.AddDocument(d);
+
+ // force merge
+ writer.Optimize();
+ // flush
+ writer.Close();
+
+ _TestUtil.CheckIndex(ram);
+
+ // only one segment in the index, so we can cast to SegmentReader
+ SegmentReader reader = (SegmentReader)IndexReader.Open(ram);
+ FieldInfos fi = reader.FieldInfos();
+ Assert.IsTrue(fi.FieldInfo("f1").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+ Assert.IsTrue(fi.FieldInfo("f2").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+
+ reader.Close();
+ ram.Close();
+ }
+
+ // Make sure first adding docs that do not omitTf for
+ // field X, then adding docs that do omitTf for that same
+ // field,
+ [Test]
+ public void TestMixedRAM()
+ {
+ Directory ram = new MockRAMDirectory();
+ Analyzer analyzer = new StandardAnalyzer();
+ IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+ writer.SetMaxBufferedDocs(10);
+ writer.SetMergeFactor(2);
+ Document d = new Document();
+
+ // this field will have Tf
+ Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
+ d.Add(f1);
+
+ // this field will NOT have Tf
+ Field f2 = new Field("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED);
+ d.Add(f2);
+
+ for (int i = 0; i < 5; i++)
+ writer.AddDocument(d);
+
+ f2.SetOmitTf(true);
+
+ for (int i = 0; i < 20; i++)
+ writer.AddDocument(d);
+
+ // force merge
+ writer.Optimize();
+
+ // flush
+ writer.Close();
+
+ _TestUtil.CheckIndex(ram);
+
+ // only one segment in the index, so we can cast to SegmentReader
+ SegmentReader reader = (SegmentReader)IndexReader.Open(ram);
+ FieldInfos fi = reader.FieldInfos();
+ Assert.IsTrue(!fi.FieldInfo("f1").omitTf_ForNUnitTest, "OmitTf field bit should not be set.");
+ Assert.IsTrue(fi.FieldInfo("f2").omitTf_ForNUnitTest, "OmitTf field bit should be set.");
+
+ reader.Close();
+ ram.Close();
+ }
+
+ private void AssertNoPrx(Directory dir)
+ {
+ string[] files = dir.List();
+ for (int i = 0; i < files.Length; i++)
+ Assert.IsFalse(files[i].EndsWith(".prx"));
+ }
+
+ // Verifies no *.prx exists when all fields omit term freq:
+ [Test]
+ public void TestNoPrxFile()
+ {
+ Directory ram = new MockRAMDirectory();
+ Analyzer analyzer = new StandardAnalyzer();
+ IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+ writer.SetMaxBufferedDocs(3);
+ writer.SetMergeFactor(2);
+ writer.SetUseCompoundFile(false);
+ Document d = new Document();
+
+ Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED);
+ f1.SetOmitTf(true);
+ d.Add(f1);
+
+ for (int i = 0; i < 30; i++)
+ writer.AddDocument(d);
+
+ writer.Commit();
+
+ AssertNoPrx(ram);
+
+ // force merge
+ writer.Optimize();
+ // flush
+ writer.Close();
+
+ AssertNoPrx(ram);
+ _TestUtil.CheckIndex(ram);
+ ram.Close();
+ }
+
+ // Test scores with one field with Term Freqs and one without, otherwise with equal content
+ [Test]
+ public void TestBasic()
+ {
+ Directory dir = new MockRAMDirectory();
+ Analyzer analyzer = new StandardAnalyzer();
+ IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
+ writer.SetMergeFactor(2);
+ writer.SetMaxBufferedDocs(2);
+ writer.SetSimilarity(new SimpleSimilarity());
+
+
+ System.Text.StringBuilder sb = new System.Text.StringBuilder(265);
+ string term = "term";
+ for (int i = 0; i < 30; i++)
+ {
+ Document d = new Document();
+ sb.Append(term).Append(" ");
+ string content = sb.ToString();
+ Field noTf = new Field("noTf", content + (i % 2 == 0 ? "" : " notf"), Field.Store.NO, Field.Index.ANALYZED);
+ noTf.SetOmitTf(true);
+ d.Add(noTf);
+
+ Field tf = new Field("tf", content + (i % 2 == 0 ? " tf" : ""), Field.Store.NO, Field.Index.ANALYZED);
+ d.Add(tf);
+
+ writer.AddDocument(d);
+ //System.out.println(d);
+ }
+
+ writer.Optimize();
+ // flush
+ writer.Close();
+ _TestUtil.CheckIndex(dir);
+
+ /*
+ * Verify the index
+ */
+ Searcher searcher = new IndexSearcher(dir);
+ searcher.SetSimilarity(new SimpleSimilarity());
+
+ Term a = new Term("noTf", term);
+ Term b = new Term("tf", term);
+ Term c = new Term("noTf", "notf");
+ Term d2 = new Term("tf", "tf");
+ TermQuery q1 = new TermQuery(a);
+ TermQuery q2 = new TermQuery(b);
+ TermQuery q3 = new TermQuery(c);
+ TermQuery q4 = new TermQuery(d2);
+
+
+ searcher.Search(q1, new AnonymousCountingHitCollector1());
+ searcher.Search(q2, new AnonymousCountingHitCollector2());
+ searcher.Search(q3, new AnonymousCountingHitCollector3());
+ searcher.Search(q4, new AnonymousCountingHitCollector4());
+
+ BooleanQuery bq = new BooleanQuery();
+ bq.Add(q1, Occur.MUST);
+ bq.Add(q4, Occur.MUST);
+
+ searcher.Search(bq, new AnonymousCountingHitCollector5());
+ Assert.IsTrue(15 == CountingHitCollector.GetCount());
+
+ searcher.Close();
+ dir.Close();
+ }
+
+ public class CountingHitCollector : HitCollector
+ {
+ static int count = 0;
+ static int sum = 0;
+ internal CountingHitCollector() { count = 0; sum = 0; }
+ override public void Collect(int doc, float score)
+ {
+ count++;
+ sum += doc; // use it to avoid any possibility of being optimized away
+ }
+
+ public static int GetCount() { return count; }
+ public static int GetSum() { return sum; }
+ }
+
+ public class AnonymousCountingHitCollector1 : CountingHitCollector
+ {
+ override public void Collect(int doc, float score)
+ {
+ //System.out.println("Q1: Doc=" + doc + " score=" + score);
+ Assert.IsTrue(score == 1.0f);
+ base.Collect(doc, score);
+ }
+ }
+ public class AnonymousCountingHitCollector2 : CountingHitCollector
+ {
+ override public void Collect(int doc, float score)
+ {
+ //System.out.println("Q1: Doc=" + doc + " score=" + score);
+ Assert.IsTrue(score == 1.0f+doc);
+ base.Collect(doc, score);
+ }
+ }
+ public class AnonymousCountingHitCollector3 : CountingHitCollector
+ {
+ override public void Collect(int doc, float score)
+ {
+ //System.out.println("Q1: Doc=" + doc + " score=" + score);
+ Assert.IsTrue(score == 1.0f);
+ Assert.IsFalse(doc % 2 == 0);
+ base.Collect(doc, score);
+ }
+ }
+ public class AnonymousCountingHitCollector4 : CountingHitCollector
+ {
+ override public void Collect(int doc, float score)
+ {
+ //System.out.println("Q1: Doc=" + doc + " score=" + score);
+ Assert.IsTrue(score == 1.0f);
+ Assert.IsTrue(doc % 2 == 0);
+ base.Collect(doc, score);
+ }
+ };
+ public class AnonymousCountingHitCollector5 : CountingHitCollector
+ {
+ override public void Collect(int doc, float score)
+ {
+ //System.out.println("BQ: Doc=" + doc + " score=" + score);
+ base.Collect(doc, score);
+ }
+ }
+ }
+}
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestParallelReader.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelReader.cs Wed Jul 29 18:04:12 2009
@@ -27,7 +27,7 @@
using RAMDirectory = Lucene.Net.Store.RAMDirectory;
using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
using BooleanQuery = Lucene.Net.Search.BooleanQuery;
-using Hits = Lucene.Net.Search.Hits;
+using ScoreDoc = Lucene.Net.Search.ScoreDoc;
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
using Query = Lucene.Net.Search.Query;
using Searcher = Lucene.Net.Search.Searcher;
@@ -78,12 +78,12 @@
ParallelReader pr = new ParallelReader();
pr.Add(IndexReader.Open(dir1));
pr.Add(IndexReader.Open(dir2));
- System.Collections.ICollection fieldNames = pr.GetFieldNames(IndexReader.FieldOption.ALL);
+ System.Collections.Generic.ICollection<string> fieldNames = pr.GetFieldNames(IndexReader.FieldOption.ALL);
Assert.AreEqual(4, fieldNames.Count);
- Assert.IsTrue(CollectionContains(fieldNames, "f1"));
- Assert.IsTrue(CollectionContains(fieldNames, "f2"));
- Assert.IsTrue(CollectionContains(fieldNames, "f3"));
- Assert.IsTrue(CollectionContains(fieldNames, "f4"));
+ Assert.IsTrue(fieldNames.Contains("f1"));
+ Assert.IsTrue(fieldNames.Contains("f2"));
+ Assert.IsTrue(fieldNames.Contains("f3"));
+ Assert.IsTrue(fieldNames.Contains("f4"));
}
[Test]
@@ -117,9 +117,9 @@
// one document only:
Directory dir2 = new MockRAMDirectory();
- IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true);
+ IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
Lucene.Net.Documents.Document d3 = new Lucene.Net.Documents.Document();
- d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+ d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.ANALYZED));
w2.AddDocument(d3);
w2.Close();
@@ -169,15 +169,15 @@
Directory dir2 = GetDir1();
// add another document to ensure that the indexes are not optimized
- IndexWriter modifier = new IndexWriter(dir1, new StandardAnalyzer());
+ IndexWriter modifier = new IndexWriter(dir1, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
Document d = new Document();
- d.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+ d.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.ANALYZED));
modifier.AddDocument(d);
modifier.Close();
-
- modifier = new IndexWriter(dir2, new StandardAnalyzer());
+
+ modifier = new IndexWriter(dir2, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
d = new Document();
- d.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.TOKENIZED));
+ d.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.ANALYZED));
modifier.AddDocument(d);
modifier.Close();
@@ -187,8 +187,8 @@
pr.Add(IndexReader.Open(dir2));
Assert.IsFalse(pr.IsOptimized());
pr.Close();
-
- modifier = new IndexWriter(dir1, new StandardAnalyzer());
+
+ modifier = new IndexWriter(dir1, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
modifier.Optimize();
modifier.Close();
@@ -198,9 +198,9 @@
// just one of the two indexes are optimized
Assert.IsFalse(pr.IsOptimized());
pr.Close();
-
-
- modifier = new IndexWriter(dir2, new StandardAnalyzer());
+
+
+ modifier = new IndexWriter(dir2, new StandardAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
modifier.Optimize();
modifier.Close();
@@ -215,14 +215,14 @@
private void QueryTest(Query query)
{
- Hits parallelHits = parallel.Search(query);
- Hits singleHits = single.Search(query);
- Assert.AreEqual(parallelHits.Length(), singleHits.Length());
- for (int i = 0; i < parallelHits.Length(); i++)
+ ScoreDoc[] parallelHits = parallel.Search(query, null, 1000).scoreDocs;
+ ScoreDoc[] singleHits = single.Search(query, null, 1000).scoreDocs;
+ Assert.AreEqual(parallelHits.Length, singleHits.Length);
+ for (int i = 0; i < parallelHits.Length; i++)
{
- Assert.AreEqual(parallelHits.Score(i), singleHits.Score(i), 0.001f);
- Lucene.Net.Documents.Document docParallel = parallelHits.Doc(i);
- Lucene.Net.Documents.Document docSingle = singleHits.Doc(i);
+ Assert.AreEqual(parallelHits[i].score, singleHits[i].score, 0.001f);
+ Lucene.Net.Documents.Document docParallel = parallel.Doc(parallelHits[i].doc);
+ Lucene.Net.Documents.Document docSingle = single.Doc(singleHits[i].doc);
Assert.AreEqual(docParallel.Get("f1"), docSingle.Get("f1"));
Assert.AreEqual(docParallel.Get("f2"), docSingle.Get("f2"));
Assert.AreEqual(docParallel.Get("f3"), docSingle.Get("f3"));
@@ -234,18 +234,18 @@
private Searcher Single()
{
Directory dir = new MockRAMDirectory();
- IndexWriter w = new IndexWriter(dir, new StandardAnalyzer(), true);
+ IndexWriter w = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
Lucene.Net.Documents.Document d1 = new Lucene.Net.Documents.Document();
- d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.TOKENIZED));
- d1.Add(new Field("f2", "v1", Field.Store.YES, Field.Index.TOKENIZED));
- d1.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.TOKENIZED));
- d1.Add(new Field("f4", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+ d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.ANALYZED));
+ d1.Add(new Field("f2", "v1", Field.Store.YES, Field.Index.ANALYZED));
+ d1.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.ANALYZED));
+ d1.Add(new Field("f4", "v1", Field.Store.YES, Field.Index.ANALYZED));
w.AddDocument(d1);
Lucene.Net.Documents.Document d2 = new Lucene.Net.Documents.Document();
- d2.Add(new Field("f1", "v2", Field.Store.YES, Field.Index.TOKENIZED));
- d2.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.TOKENIZED));
- d2.Add(new Field("f3", "v2", Field.Store.YES, Field.Index.TOKENIZED));
- d2.Add(new Field("f4", "v2", Field.Store.YES, Field.Index.TOKENIZED));
+ d2.Add(new Field("f1", "v2", Field.Store.YES, Field.Index.ANALYZED));
+ d2.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.ANALYZED));
+ d2.Add(new Field("f3", "v2", Field.Store.YES, Field.Index.ANALYZED));
+ d2.Add(new Field("f4", "v2", Field.Store.YES, Field.Index.ANALYZED));
w.AddDocument(d2);
w.Close();
@@ -266,14 +266,14 @@
private Directory GetDir1()
{
Directory dir1 = new MockRAMDirectory();
- IndexWriter w1 = new IndexWriter(dir1, new StandardAnalyzer(), true);
+ IndexWriter w1 = new IndexWriter(dir1, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
Lucene.Net.Documents.Document d1 = new Lucene.Net.Documents.Document();
- d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.TOKENIZED));
- d1.Add(new Field("f2", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+ d1.Add(new Field("f1", "v1", Field.Store.YES, Field.Index.ANALYZED));
+ d1.Add(new Field("f2", "v1", Field.Store.YES, Field.Index.ANALYZED));
w1.AddDocument(d1);
Lucene.Net.Documents.Document d2 = new Lucene.Net.Documents.Document();
- d2.Add(new Field("f1", "v2", Field.Store.YES, Field.Index.TOKENIZED));
- d2.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.TOKENIZED));
+ d2.Add(new Field("f1", "v2", Field.Store.YES, Field.Index.ANALYZED));
+ d2.Add(new Field("f2", "v2", Field.Store.YES, Field.Index.ANALYZED));
w1.AddDocument(d2);
w1.Close();
return dir1;
@@ -282,28 +282,17 @@
private Directory GetDir2()
{
Directory dir2 = new RAMDirectory();
- IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true);
+ IndexWriter w2 = new IndexWriter(dir2, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
Lucene.Net.Documents.Document d3 = new Lucene.Net.Documents.Document();
- d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.TOKENIZED));
- d3.Add(new Field("f4", "v1", Field.Store.YES, Field.Index.TOKENIZED));
+ d3.Add(new Field("f3", "v1", Field.Store.YES, Field.Index.ANALYZED));
+ d3.Add(new Field("f4", "v1", Field.Store.YES, Field.Index.ANALYZED));
w2.AddDocument(d3);
Lucene.Net.Documents.Document d4 = new Lucene.Net.Documents.Document();
- d4.Add(new Field("f3", "v2", Field.Store.YES, Field.Index.TOKENIZED));
- d4.Add(new Field("f4", "v2", Field.Store.YES, Field.Index.TOKENIZED));
+ d4.Add(new Field("f3", "v2", Field.Store.YES, Field.Index.ANALYZED));
+ d4.Add(new Field("f4", "v2", Field.Store.YES, Field.Index.ANALYZED));
w2.AddDocument(d4);
w2.Close();
return dir2;
}
-
- public static bool CollectionContains(System.Collections.ICollection col, System.String val)
- {
- for (System.Collections.IEnumerator iterator = col.GetEnumerator(); iterator.MoveNext(); )
- {
- System.String s = (System.String)iterator.Current;
- if (s == val)
- return true;
- }
- return false;
- }
}
-}
+}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestParallelTermEnum.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestParallelTermEnum.cs Wed Jul 29 18:04:12 2009
@@ -44,22 +44,22 @@
Lucene.Net.Documents.Document doc;
RAMDirectory rd1 = new RAMDirectory();
- IndexWriter iw1 = new IndexWriter(rd1, new SimpleAnalyzer(), true);
+ IndexWriter iw1 = new IndexWriter(rd1, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
doc = new Lucene.Net.Documents.Document();
- doc.Add(new Field("field1", "the quick brown fox jumps", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.TOKENIZED));
- doc.Add(new Field("field2", "the quick brown fox jumps", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.TOKENIZED));
- doc.Add(new Field("field4", "", Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.TOKENIZED));
+ doc.Add(new Field("field1", "the quick brown fox jumps", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
+ doc.Add(new Field("field2", "the quick brown fox jumps", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
+ doc.Add(new Field("field4", "", Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.ANALYZED));
iw1.AddDocument(doc);
iw1.Close();
RAMDirectory rd2 = new RAMDirectory();
- IndexWriter iw2 = new IndexWriter(rd2, new SimpleAnalyzer(), true);
+ IndexWriter iw2 = new IndexWriter(rd2, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);
doc = new Lucene.Net.Documents.Document();
- doc.Add(new Field("field0", "", Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.TOKENIZED));
- doc.Add(new Field("field1", "the fox jumps over the lazy dog", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.TOKENIZED));
- doc.Add(new Field("field3", "the fox jumps over the lazy dog", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.TOKENIZED));
+ doc.Add(new Field("field0", "", Lucene.Net.Documents.Field.Store.NO, Lucene.Net.Documents.Field.Index.ANALYZED));
+ doc.Add(new Field("field1", "the fox jumps over the lazy dog", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
+ doc.Add(new Field("field3", "the fox jumps over the lazy dog", Lucene.Net.Documents.Field.Store.YES, Lucene.Net.Documents.Field.Index.ANALYZED));
iw2.AddDocument(doc);
iw2.Close();
Modified: incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Index/TestPayloads.cs?rev=798995&r1=798994&r2=798995&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Index/TestPayloads.cs Wed Jul 29 18:04:12 2009
@@ -31,6 +31,7 @@
using TokenStream = Lucene.Net.Analysis.TokenStream;
using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
using WhitespaceTokenizer = Lucene.Net.Analysis.WhitespaceTokenizer;
+using UnicodeUtil = Lucene.Net.Util.UnicodeUtil;
namespace Lucene.Net.Index
{
@@ -149,18 +150,18 @@
{
Directory ram = new RAMDirectory();
PayloadAnalyzer analyzer = new PayloadAnalyzer();
- IndexWriter writer = new IndexWriter(ram, analyzer, true);
+ IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
Document d = new Document();
// this field won't have any payloads
- d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.ANALYZED));
// this field will have payloads in all docs, however not for all term positions,
// so this field is used to check if the DocumentWriter correctly enables the payloads bit
// even if only some term positions have payloads
- d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
- d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.ANALYZED));
+ d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.ANALYZED));
// this field is used to verify if the SegmentMerger enables payloads for a field if it has payloads
// enabled in only some documents
- d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.ANALYZED));
// only add payload data for field f2
analyzer.SetPayloadData("f2", 1, System.Text.Encoding.UTF8.GetBytes("somedata"), 0, 1);
writer.AddDocument(d);
@@ -177,12 +178,12 @@
// now we add another document which has payloads for field f3 and verify if the SegmentMerger
// enabled payloads for that field
- writer = new IndexWriter(ram, analyzer, true);
+ writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
d = new Document();
- d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.TOKENIZED));
- d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
- d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.TOKENIZED));
- d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field("f1", "This field has no payloads", Field.Store.NO, Field.Index.ANALYZED));
+ d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.ANALYZED));
+ d.Add(new Field("f2", "This field has payloads in all docs", Field.Store.NO, Field.Index.ANALYZED));
+ d.Add(new Field("f3", "This field has payloads in some docs", Field.Store.NO, Field.Index.ANALYZED));
// add payload data for field f2 and f3
analyzer.SetPayloadData("f2", System.Text.Encoding.UTF8.GetBytes("somedata"), 0, 1);
analyzer.SetPayloadData("f3", System.Text.Encoding.UTF8.GetBytes("somedata"), 0, 3);
@@ -221,7 +222,7 @@
private void PerformTest(Directory dir)
{
PayloadAnalyzer analyzer = new PayloadAnalyzer();
- IndexWriter writer = new IndexWriter(dir, analyzer, true);
+ IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
// should be in sync with value in TermInfosWriter
int skipInterval = 16;
@@ -245,7 +246,7 @@
byte[] payloadData = GenerateRandomData(payloadDataLength);
Document d = new Document();
- d.Add(new Field(fieldName, content, Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field(fieldName, content, Field.Store.NO, Field.Index.ANALYZED));
// add the same document multiple times to have the same payload lengths for all
// occurrences within two consecutive skip intervals
int offset = 0;
@@ -370,11 +371,11 @@
// test long payload
analyzer = new PayloadAnalyzer();
- writer = new IndexWriter(dir, analyzer, true);
+ writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED);
System.String singleTerm = "lucene";
d = new Document();
- d.Add(new Field(fieldName, singleTerm, Field.Store.NO, Field.Index.TOKENIZED));
+ d.Add(new Field(fieldName, singleTerm, Field.Store.NO, Field.Index.ANALYZED));
// add a payload whose length is greater than the buffer size of BufferedIndexOutput
payloadData = GenerateRandomData(2000);
analyzer.SetPayloadData(fieldName, payloadData, 100, 1500);
@@ -567,10 +568,11 @@
this.offset = offset;
}
- public override Token Next(Token token)
+ public override Token Next(Token reusableToken)
{
- token = input.Next(token);
- if (token != null)
+ System.Diagnostics.Debug.Assert(reusableToken != null);
+ Token nextToken = input.Next(reusableToken);
+ if (nextToken != null)
{
if (offset + length <= data.Length)
{
@@ -578,18 +580,18 @@
if (p == null)
{
p = new Payload();
- token.SetPayload(p);
+ nextToken.SetPayload(p);
}
p.SetData(data, offset, length);
offset += length;
}
else
{
- token.SetPayload(null);
+ nextToken.SetPayload(null);
}
}
- return token;
+ return nextToken;
}
}
@@ -601,7 +603,7 @@
ByteArrayPool pool = new ByteArrayPool(numThreads, 5);
Directory dir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer());
+ IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED);
System.String field = "test";
SupportClass.ThreadClass[] ingesters = new SupportClass.ThreadClass[numThreads];
@@ -633,8 +635,9 @@
for (int i = 0; i < freq; i++)
{
tp.NextPosition();
- System.String s = System.Text.Encoding.UTF8.GetString(tp.GetPayload(new byte[5], 0));
- Assert.AreEqual(s, terms.Term().text_ForNUnitTest);
+ //System.String s = System.Text.Encoding.UTF8.GetString(tp.GetPayload(new byte[5], 0));
+ //Assert.AreEqual(s, terms.Term().text_ForNUnitTest);
+ Assert.AreEqual(pool.BytesToString(tp.GetPayload(new byte[5], 0)), terms.Term().text_ForNUnitTest);
}
}
tp.Close();
@@ -650,22 +653,23 @@
private byte[] payload;
private bool first;
private ByteArrayPool pool;
-
+ private string term;
internal PoolingPayloadTokenStream(ByteArrayPool pool)
{
this.pool = pool;
payload = pool.Get();
Lucene.Net.Index.TestPayloads.GenerateRandomData(payload);
+ term = pool.BytesToString(payload);
first = true;
}
- public override Token Next()
+ public override Token Next(Token reusableToken)
{
if (!first)
return null;
- Token t = new Token(System.Text.Encoding.UTF8.GetString(payload), 0, 0);
- t.SetPayload(new Payload(payload));
- return t;
+ reusableToken.Reinit(term, 0, 0);
+ reusableToken.SetPayload(new Payload(payload));
+ return reusableToken;
}
public override void Close()
@@ -686,7 +690,25 @@
pool.Add(new byte[size]);
}
}
-
+
+ private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
+
+ internal string BytesToString(byte[] bytes)
+ {
+ lock (this)
+ {
+ string s = System.Text.Encoding.Default.GetString(bytes);
+ UnicodeUtil.UTF16toUTF8(s, 0, s.Length, utf8Result);
+ try{
+ return System.Text.Encoding.UTF8.GetString(utf8Result.result, 0, utf8Result.length);
+ }
+ catch (System.Text.DecoderFallbackException)
+ {
+ return null;
+ }
+ }
+ }
+
internal virtual byte[] Get()
{
lock (this)