You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ar...@apache.org on 2008/07/15 23:44:10 UTC
svn commit: r677059 [2/19] - in /incubator/lucene.net/trunk/C#/src: ./
Demo/DeleteFiles/ Demo/DemoLib/ Demo/IndexFiles/ Demo/IndexHtml/
Demo/SearchFiles/ Lucene.Net/ Lucene.Net/Analysis/ Lucene.Net/Index/
Lucene.Net/Search/ Lucene.Net/Search/Function/ ...
Added: incubator/lucene.net/trunk/C#/src/Test/Analysis/TeeSinkTokenTest.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TeeSinkTokenTest.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TeeSinkTokenTest.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TeeSinkTokenTest.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using NUnit.Framework;
+
+using StandardFilter = Lucene.Net.Analysis.Standard.StandardFilter;
+using StandardTokenizer = Lucene.Net.Analysis.Standard.StandardTokenizer;
+using English = Lucene.Net.Util.English;
+
+namespace Lucene.Net.Analysis
+{
+
+ /// <summary> tests for the TeeTokenFilter and SinkTokenizer</summary>
+ [TestFixture]
+ public class TeeSinkTokenTest
+ {
+ private class AnonymousClassSinkTokenizer : SinkTokenizer
+ {
+ private void InitBlock(TeeSinkTokenTest enclosingInstance)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+ private TeeSinkTokenTest enclosingInstance;
+ public TeeSinkTokenTest Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+ internal AnonymousClassSinkTokenizer(TeeSinkTokenTest enclosingInstance, System.Collections.IList Param1) : base(Param1)
+ {
+ InitBlock(enclosingInstance);
+ }
+ public override void Add(Token t)
+ {
+ if (t != null && t.TermText().ToUpper().Equals("The".ToUpper()))
+ {
+ base.Add(t);
+ }
+ }
+ }
+
+ private class AnonymousClassSinkTokenizer1 : SinkTokenizer
+ {
+ private void InitBlock(TeeSinkTokenTest enclosingInstance)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+ private TeeSinkTokenTest enclosingInstance;
+ public TeeSinkTokenTest Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+ internal AnonymousClassSinkTokenizer1(TeeSinkTokenTest enclosingInstance, System.Collections.IList Param1) : base(Param1)
+ {
+ InitBlock(enclosingInstance);
+ }
+ public override void Add(Token t)
+ {
+ if (t != null && t.TermText().ToUpper().Equals("The".ToUpper()))
+ {
+ base.Add(t);
+ }
+ }
+ }
+
+ private class AnonymousClassSinkTokenizer2 : SinkTokenizer
+ {
+ private void InitBlock(TeeSinkTokenTest enclosingInstance)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+ private TeeSinkTokenTest enclosingInstance;
+ public TeeSinkTokenTest Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+ internal AnonymousClassSinkTokenizer2(TeeSinkTokenTest enclosingInstance, System.Collections.IList Param1) : base(Param1)
+ {
+ InitBlock(enclosingInstance);
+ }
+ public override void Add(Token t)
+ {
+ if (t != null && t.TermText().ToUpper().Equals("Dogs".ToUpper()))
+ {
+ base.Add(t);
+ }
+ }
+ }
+ protected internal System.Text.StringBuilder buffer1;
+ protected internal System.Text.StringBuilder buffer2;
+ protected internal System.String[] tokens1;
+ protected internal System.String[] tokens2;
+
+ [SetUp]
+ public virtual void SetUp()
+ {
+ tokens1 = new System.String[]{"The", "quick", "Burgundy", "Fox", "jumped", "over", "the", "lazy", "Red", "Dogs"};
+ tokens2 = new System.String[]{"The", "Lazy", "Dogs", "should", "stay", "on", "the", "porch"};
+ buffer1 = new System.Text.StringBuilder();
+
+ for (int i = 0; i < tokens1.Length; i++)
+ {
+ buffer1.Append(tokens1[i]).Append(' ');
+ }
+ buffer2 = new System.Text.StringBuilder();
+ for (int i = 0; i < tokens2.Length; i++)
+ {
+ buffer2.Append(tokens2[i]).Append(' ');
+ }
+ }
+
+ [TearDown]
+ public virtual void TearDown()
+ {
+
+ }
+
+ [Test]
+ public virtual void Test()
+ {
+
+ SinkTokenizer sink1 = new AnonymousClassSinkTokenizer(this, null);
+ TokenStream source = new TeeTokenFilter(new WhitespaceTokenizer(new System.IO.StringReader(buffer1.ToString())), sink1);
+ Token token = null;
+ int i = 0;
+ while ((token = source.Next()) != null)
+ {
+ Assert.IsTrue(token.TermText().Equals(tokens1[i]) == true, token.TermText() + " is not equal to " + tokens1[i]);
+ i++;
+ }
+ Assert.IsTrue(i == tokens1.Length, i + " does not equal: " + tokens1.Length);
+ Assert.IsTrue(sink1.GetTokens().Count == 2, "sink1 Size: " + sink1.GetTokens().Count + " is not: " + 2);
+ i = 0;
+ while ((token = sink1.Next()) != null)
+ {
+ Assert.IsTrue(token.TermText().ToUpper().Equals("The".ToUpper()) == true, token.TermText() + " is not equal to " + "The");
+ i++;
+ }
+ Assert.IsTrue(i == sink1.GetTokens().Count, i + " does not equal: " + sink1.GetTokens().Count);
+ }
+
+ [Test]
+ public virtual void TestMultipleSources()
+ {
+ SinkTokenizer theDetector = new AnonymousClassSinkTokenizer1(this, null);
+ SinkTokenizer dogDetector = new AnonymousClassSinkTokenizer2(this, null);
+ TokenStream source1 = new CachingTokenFilter(new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new System.IO.StringReader(buffer1.ToString())), theDetector), dogDetector));
+ TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new System.IO.StringReader(buffer2.ToString())), theDetector), dogDetector);
+ Token token = null;
+ int i = 0;
+ while ((token = source1.Next()) != null)
+ {
+ Assert.IsTrue(token.TermText().Equals(tokens1[i]) == true, token.TermText() + " is not equal to " + tokens1[i]);
+ i++;
+ }
+ Assert.IsTrue(i == tokens1.Length, i + " does not equal: " + tokens1.Length);
+ Assert.IsTrue(theDetector.GetTokens().Count == 2, "theDetector Size: " + theDetector.GetTokens().Count + " is not: " + 2);
+ Assert.IsTrue(dogDetector.GetTokens().Count == 1, "dogDetector Size: " + dogDetector.GetTokens().Count + " is not: " + 1);
+ i = 0;
+ while ((token = source2.Next()) != null)
+ {
+ Assert.IsTrue(token.TermText().Equals(tokens2[i]) == true, token.TermText() + " is not equal to " + tokens2[i]);
+ i++;
+ }
+ Assert.IsTrue(i == tokens2.Length, i + " does not equal: " + tokens2.Length);
+ Assert.IsTrue(theDetector.GetTokens().Count == 4, "theDetector Size: " + theDetector.GetTokens().Count + " is not: " + 4);
+ Assert.IsTrue(dogDetector.GetTokens().Count == 2, "dogDetector Size: " + dogDetector.GetTokens().Count + " is not: " + 2);
+ i = 0;
+ while ((token = theDetector.Next()) != null)
+ {
+ Assert.IsTrue(token.TermText().ToUpper().Equals("The".ToUpper()) == true, token.TermText() + " is not equal to " + "The");
+ i++;
+ }
+ Assert.IsTrue(i == theDetector.GetTokens().Count, i + " does not equal: " + theDetector.GetTokens().Count);
+ i = 0;
+ while ((token = dogDetector.Next()) != null)
+ {
+ Assert.IsTrue(token.TermText().ToUpper().Equals("Dogs".ToUpper()) == true, token.TermText() + " is not equal to " + "Dogs");
+ i++;
+ }
+ Assert.IsTrue(i == dogDetector.GetTokens().Count, i + " does not equal: " + dogDetector.GetTokens().Count);
+ source1.Reset();
+ TokenStream lowerCasing = new LowerCaseFilter(source1);
+ i = 0;
+ while ((token = lowerCasing.Next()) != null)
+ {
+ Assert.IsTrue(token.TermText().Equals(tokens1[i].ToLower()) == true, token.TermText() + " is not equal to " + tokens1[i].ToLower());
+ i++;
+ }
+ Assert.IsTrue(i == tokens1.Length, i + " does not equal: " + tokens1.Length);
+ }
+
+ /// <summary> Not an explicit test, just useful to print out some info on performance
+ ///
+ /// </summary>
+ /// <throws> Exception </throws>
+ [Test]
+ public virtual void TestPerformance()
+ {
+ int[] tokCount = new int[]{100, 500, 1000, 2000, 5000, 10000};
+ int[] modCounts = new int[]{1, 2, 5, 10, 20, 50, 100, 200, 500};
+ for (int k = 0; k < tokCount.Length; k++)
+ {
+ System.Text.StringBuilder buffer = new System.Text.StringBuilder();
+ System.Console.Out.WriteLine("-----Tokens: " + tokCount[k] + "-----");
+ for (int i = 0; i < tokCount[k]; i++)
+ {
+ buffer.Append(English.IntToEnglish(i).ToUpper()).Append(' ');
+ }
+ //make sure we produce the same tokens
+ ModuloSinkTokenizer sink = new ModuloSinkTokenizer(this, tokCount[k], 100);
+ Token next = new Token();
+ TokenStream result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), sink);
+ while ((next = result.Next(next)) != null)
+ {
+ }
+ result = new ModuloTokenFilter(this, new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), 100);
+ next = new Token();
+ System.Collections.IList tmp = new System.Collections.ArrayList();
+ while ((next = result.Next(next)) != null)
+ {
+ tmp.Add(next.Clone());
+ }
+ System.Collections.IList sinkList = sink.GetTokens();
+ Assert.IsTrue(tmp.Count == sinkList.Count, "tmp Size: " + tmp.Count + " is not: " + sinkList.Count);
+ for (int i = 0; i < tmp.Count; i++)
+ {
+ Token tfTok = (Token) tmp[i];
+ Token sinkTok = (Token) sinkList[i];
+ Assert.IsTrue(tfTok.TermText().Equals(sinkTok.TermText()) == true, tfTok.TermText() + " is not equal to " + sinkTok.TermText() + " at token: " + i);
+ }
+ //simulate two fields, each being analyzed once, for 20 documents
+
+ for (int j = 0; j < modCounts.Length; j++)
+ {
+ int tfPos = 0;
+ long start = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
+ for (int i = 0; i < 20; i++)
+ {
+ next = new Token();
+ result = new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString())));
+ while ((next = result.Next(next)) != null)
+ {
+ tfPos += next.GetPositionIncrement();
+ }
+ next = new Token();
+ result = new ModuloTokenFilter(this, new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), modCounts[j]);
+ while ((next = result.Next(next)) != null)
+ {
+ tfPos += next.GetPositionIncrement();
+ }
+ }
+ long finish = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
+ System.Console.Out.WriteLine("ModCount: " + modCounts[j] + " Two fields took " + (finish - start) + " ms");
+ int sinkPos = 0;
+ //simulate one field with one sink
+ start = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
+ for (int i = 0; i < 20; i++)
+ {
+ sink = new ModuloSinkTokenizer(this, tokCount[k], modCounts[j]);
+ next = new Token();
+ result = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new System.IO.StringReader(buffer.ToString()))), sink);
+ while ((next = result.Next(next)) != null)
+ {
+ sinkPos += next.GetPositionIncrement();
+ }
+ //System.out.println("Modulo--------");
+ result = sink;
+ while ((next = result.Next(next)) != null)
+ {
+ sinkPos += next.GetPositionIncrement();
+ }
+ }
+ finish = (System.DateTime.Now.Ticks - 621355968000000000) / 10000;
+ System.Console.Out.WriteLine("ModCount: " + modCounts[j] + " Tee fields took " + (finish - start) + " ms");
+ Assert.IsTrue(sinkPos == tfPos, sinkPos + " does not equal: " + tfPos);
+ }
+ System.Console.Out.WriteLine("- End Tokens: " + tokCount[k] + "-----");
+ }
+ }
+
+
+ internal class ModuloTokenFilter : TokenFilter
+ {
+ private void InitBlock(TeeSinkTokenTest enclosingInstance)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+ private TeeSinkTokenTest enclosingInstance;
+ public TeeSinkTokenTest Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+
+ internal int modCount;
+
+ internal ModuloTokenFilter(TeeSinkTokenTest enclosingInstance, TokenStream input, int mc):base(input)
+ {
+ InitBlock(enclosingInstance);
+ modCount = mc;
+ }
+
+ internal int count = 0;
+
+ //return every 100 tokens
+ public override Token Next(Token result)
+ {
+
+ while ((result = input.Next(result)) != null && count % modCount != 0)
+ {
+ count++;
+ }
+ count++;
+ return result;
+ }
+ }
+
+ internal class ModuloSinkTokenizer : SinkTokenizer
+ {
+ private void InitBlock(TeeSinkTokenTest enclosingInstance)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+ private TeeSinkTokenTest enclosingInstance;
+ public TeeSinkTokenTest Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+ internal int count = 0;
+ internal int modCount;
+
+
+ internal ModuloSinkTokenizer(TeeSinkTokenTest enclosingInstance, int numToks, int mc)
+ {
+ InitBlock(enclosingInstance);
+ modCount = mc;
+ lst = new System.Collections.ArrayList(numToks % mc);
+ }
+
+ public override void Add(Token t)
+ {
+ if (t != null && count % modCount == 0)
+ {
+ lst.Add(t.Clone());
+ }
+ count++;
+ }
+ }
+ }
+}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestAnalyzers.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestAnalyzers.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestAnalyzers.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestAnalyzers.cs Tue Jul 15 14:44:04 2008
@@ -19,15 +19,17 @@
using NUnit.Framework;
+using Payload = Lucene.Net.Index.Payload;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
namespace Lucene.Net.Analysis
{
[TestFixture]
- public class TestAnalyzers
+ public class TestAnalyzers : LuceneTestCase
{
-
- public virtual void AssertAnalyzesTo(Analyzer a, System.String input, System.String[] output)
+ public virtual void AssertAnalyzesTo(Analyzer a, System.String input, System.String[] output)
{
TokenStream ts = a.TokenStream("dummy", new System.IO.StringReader(input));
for (int i = 0; i < output.Length; i++)
@@ -40,7 +42,7 @@
ts.Close();
}
- [Test]
+ [Test]
public virtual void TestSimple()
{
Analyzer a = new SimpleAnalyzer();
@@ -54,7 +56,7 @@
AssertAnalyzesTo(a, "\"QUOTED\" word", new System.String[]{"quoted", "word"});
}
- [Test]
+ [Test]
public virtual void TestNull()
{
Analyzer a = new WhitespaceAnalyzer();
@@ -68,12 +70,97 @@
AssertAnalyzesTo(a, "\"QUOTED\" word", new System.String[]{"\"QUOTED\"", "word"});
}
- [Test]
+ [Test]
public virtual void TestStop()
{
Analyzer a = new StopAnalyzer();
AssertAnalyzesTo(a, "foo bar FOO BAR", new System.String[]{"foo", "bar", "foo", "bar"});
AssertAnalyzesTo(a, "foo a bar such FOO THESE BAR", new System.String[]{"foo", "bar", "foo", "bar"});
}
+
+ internal virtual void VerifyPayload(TokenStream ts)
+ {
+ Token t = new Token();
+ for (byte b = 1; ; b++)
+ {
+ t.Clear();
+ t = ts.Next(t);
+ if (t == null)
+ break;
+ // System.out.println("id="+System.identityHashCode(t) + " " + t);
+ // System.out.println("payload=" + (int)t.getPayload().toByteArray()[0]);
+ Assert.AreEqual(b, t.GetPayload().ToByteArray()[0]);
+ }
+ }
+
+ // Make sure old style next() calls result in a new copy of payloads
+ [Test]
+ public virtual void TestPayloadCopy()
+ {
+ System.String s = "how now brown cow";
+ TokenStream ts;
+ ts = new WhitespaceTokenizer(new System.IO.StringReader(s));
+ ts = new BuffTokenFilter(ts);
+ ts = new PayloadSetter(ts);
+ VerifyPayload(ts);
+
+ ts = new WhitespaceTokenizer(new System.IO.StringReader(s));
+ ts = new PayloadSetter(ts);
+ ts = new BuffTokenFilter(ts);
+ VerifyPayload(ts);
+ }
+ }
+
+ class BuffTokenFilter : TokenFilter
+ {
+ internal System.Collections.IList lst;
+
+ public BuffTokenFilter(TokenStream input) : base(input)
+ {
+ }
+
+ public override Token Next()
+ {
+ if (lst == null)
+ {
+ lst = new System.Collections.ArrayList();
+ for (; ; )
+ {
+ Token t = input.Next();
+ if (t == null)
+ break;
+ lst.Add(t);
+ }
+ }
+ System.Object tempObject;
+ tempObject = lst[0];
+ lst.RemoveAt(0);
+ return lst.Count == 0 ? null : (Token) tempObject;
+ }
+ }
+
+ class PayloadSetter : TokenFilter
+ {
+ private void InitBlock()
+ {
+ p = new Payload(data, 0, 1);
+ }
+ public PayloadSetter(TokenStream input) : base(input)
+ {
+ InitBlock();
+ }
+
+ internal byte[] data = new byte[1];
+ internal Payload p;
+
+ public override Token Next(Token target)
+ {
+ target = input.Next(target);
+ if (target == null)
+ return null;
+ target.SetPayload(p); // reuse the payload / byte[]
+ data[0]++;
+ return target;
+ }
}
}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestCachingTokenFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestCachingTokenFilter.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestCachingTokenFilter.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestCachingTokenFilter.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using NUnit.Framework;
+
+using Document = Lucene.Net.Documents.Document;
+using Field = Lucene.Net.Documents.Field;
+using TermVector = Lucene.Net.Documents.Field.TermVector;
+using IndexReader = Lucene.Net.Index.IndexReader;
+using IndexWriter = Lucene.Net.Index.IndexWriter;
+using Term = Lucene.Net.Index.Term;
+using TermPositions = Lucene.Net.Index.TermPositions;
+using Directory = Lucene.Net.Store.Directory;
+using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+namespace Lucene.Net.Analysis
+{
+
+ public class TestCachingTokenFilter : LuceneTestCase
+ {
+ private class AnonymousClassTokenStream : TokenStream
+ {
+ public AnonymousClassTokenStream(TestCachingTokenFilter enclosingInstance)
+ {
+ InitBlock(enclosingInstance);
+ }
+ private void InitBlock(TestCachingTokenFilter enclosingInstance)
+ {
+ this.enclosingInstance = enclosingInstance;
+ }
+ private TestCachingTokenFilter enclosingInstance;
+ public TestCachingTokenFilter Enclosing_Instance
+ {
+ get
+ {
+ return enclosingInstance;
+ }
+
+ }
+ private int index = 0;
+
+ public override Token Next()
+ {
+ if (index == Enclosing_Instance.tokens.Length)
+ {
+ return null;
+ }
+ else
+ {
+ return new Token(Enclosing_Instance.tokens[index++], 0, 0);
+ }
+ }
+ }
+ private System.String[] tokens = new System.String[]{"term1", "term2", "term3", "term2"};
+
+ [Test]
+ public virtual void TestCaching()
+ {
+ Directory dir = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(dir, new SimpleAnalyzer());
+ Document doc = new Document();
+ TokenStream stream = new AnonymousClassTokenStream(this);
+
+ stream = new CachingTokenFilter(stream);
+
+ doc.Add(new Field("preanalyzed", stream, TermVector.NO));
+
+ // 1) we consume all tokens twice before we add the doc to the index
+ CheckTokens(stream);
+ stream.Reset();
+ CheckTokens(stream);
+
+ // 2) now add the document to the index and verify if all tokens are indexed
+ // don't reset the stream here, the DocumentWriter should do that implicitly
+ writer.AddDocument(doc);
+ writer.Close();
+
+ IndexReader reader = IndexReader.Open(dir);
+ TermPositions termPositions = reader.TermPositions(new Term("preanalyzed", "term1"));
+ Assert.IsTrue(termPositions.Next());
+ Assert.AreEqual(1, termPositions.Freq());
+ Assert.AreEqual(0, termPositions.NextPosition());
+
+ termPositions.Seek(new Term("preanalyzed", "term2"));
+ Assert.IsTrue(termPositions.Next());
+ Assert.AreEqual(2, termPositions.Freq());
+ Assert.AreEqual(1, termPositions.NextPosition());
+ Assert.AreEqual(3, termPositions.NextPosition());
+
+ termPositions.Seek(new Term("preanalyzed", "term3"));
+ Assert.IsTrue(termPositions.Next());
+ Assert.AreEqual(1, termPositions.Freq());
+ Assert.AreEqual(2, termPositions.NextPosition());
+ reader.Close();
+
+ // 3) reset stream and consume tokens again
+ stream.Reset();
+ CheckTokens(stream);
+ }
+
+ private void CheckTokens(TokenStream stream)
+ {
+ int count = 0;
+ Token token;
+ while ((token = stream.Next()) != null)
+ {
+ Assert.IsTrue(count < tokens.Length);
+ Assert.AreEqual(tokens[count], token.TermText());
+ count++;
+ }
+
+ Assert.AreEqual(tokens.Length, count);
+ }
+ }
+}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestCharArraySet.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestCharArraySet.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestCharArraySet.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestCharArraySet.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using NUnit.Framework;
+
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+namespace Lucene.Net.Analysis
+{
+
+ [TestFixture]
+ public class TestCharArraySet : LuceneTestCase
+ {
+ [Test]
+ public virtual void TestRehash()
+ {
+ CharArraySet cas = new CharArraySet(0, true);
+ for (int i = 0; i < StopAnalyzer.ENGLISH_STOP_WORDS.Length; i++)
+ cas.Add(StopAnalyzer.ENGLISH_STOP_WORDS[i]);
+ Assert.AreEqual(StopAnalyzer.ENGLISH_STOP_WORDS.Length, cas.Count);
+ for (int i = 0; i < StopAnalyzer.ENGLISH_STOP_WORDS.Length; i++)
+ Assert.IsTrue(cas.Contains(StopAnalyzer.ENGLISH_STOP_WORDS[i]));
+ }
+
+ [Test]
+ public virtual void TestNonZeroOffset()
+ {
+ System.String[] words = new System.String[]{"Hello", "World", "this", "is", "a", "test"};
+ char[] findme = "xthisy".ToCharArray();
+ CharArraySet set_Renamed = new CharArraySet(10, true);
+ for (int i = 0; i < words.Length; i++) { set_Renamed.Add(words[i]); }
+ Assert.IsTrue(set_Renamed.Contains(findme, 1, 4));
+ Assert.IsTrue(set_Renamed.Contains(new System.String(findme, 1, 4)));
+ }
+ }
+}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestISOLatin1AccentFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestISOLatin1AccentFilter.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestISOLatin1AccentFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestISOLatin1AccentFilter.cs Tue Jul 15 14:44:04 2008
@@ -1,4 +1,4 @@
-/*
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -19,12 +19,15 @@
using NUnit.Framework;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
namespace Lucene.Net.Analysis
{
+
[TestFixture]
- public class TestISOLatin1AccentFilter
+ public class TestISOLatin1AccentFilter : LuceneTestCase
{
- [Test]
+ [Test]
public virtual void TestU()
{
TokenStream stream = new WhitespaceTokenizer(new System.IO.StringReader("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à à à à à à à à à Šà à à à à à Ÿ à á â ã ä å æ ç è é ê ë ì à î ï ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ"));
Modified: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestKeywordAnalyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestKeywordAnalyzer.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestKeywordAnalyzer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestKeywordAnalyzer.cs Tue Jul 15 14:44:04 2008
@@ -19,31 +19,37 @@
using NUnit.Framework;
-using IndexWriter = Lucene.Net.Index.IndexWriter;
-using RAMDirectory = Lucene.Net.Store.RAMDirectory;
using Document = Lucene.Net.Documents.Document;
using Field = Lucene.Net.Documents.Field;
+using IndexReader = Lucene.Net.Index.IndexReader;
+using IndexWriter = Lucene.Net.Index.IndexWriter;
+using Term = Lucene.Net.Index.Term;
+using TermDocs = Lucene.Net.Index.TermDocs;
+using QueryParser = Lucene.Net.QueryParsers.QueryParser;
+using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using Hits = Lucene.Net.Search.Hits;
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
using Query = Lucene.Net.Search.Query;
-using Hits = Lucene.Net.Search.Hits;
-using QueryParser = Lucene.Net.QueryParsers.QueryParser;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Analysis
{
+
[TestFixture]
- public class TestKeywordAnalyzer
+ public class TestKeywordAnalyzer : LuceneTestCase
{
private RAMDirectory directory;
private IndexSearcher searcher;
- [SetUp]
- public virtual void SetUp()
+ [SetUp]
+ public override void SetUp()
{
+ base.SetUp();
directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory, new SimpleAnalyzer(), true);
- Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
+ Document doc = new Document();
doc.Add(new Field("partnum", "Q36", Field.Store.YES, Field.Index.UN_TOKENIZED));
doc.Add(new Field("description", "Illidium Space Modulator", Field.Store.YES, Field.Index.TOKENIZED));
writer.AddDocument(doc);
@@ -53,12 +59,12 @@
searcher = new IndexSearcher(directory);
}
- [Test]
+ [Test]
public virtual void TestPerFieldAnalyzer()
{
PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new SimpleAnalyzer());
analyzer.AddAnalyzer("partnum", new KeywordAnalyzer());
-
+
Lucene.Net.QueryParsers.QueryParser queryParser = new Lucene.Net.QueryParsers.QueryParser("description", analyzer);
Query query = queryParser.Parse("partnum:Q36 AND SPACE");
@@ -66,5 +72,25 @@
Assert.AreEqual("+partnum:Q36 +space", query.ToString("description"), "Q36 kept as-is");
Assert.AreEqual(1, hits.Length(), "doc found!");
}
+
+ [Test]
+ public virtual void TestMutipleDocument()
+ {
+ RAMDirectory dir = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(dir, new KeywordAnalyzer(), true);
+ Document doc = new Document();
+ doc.Add(new Field("partnum", "Q36", Field.Store.YES, Field.Index.TOKENIZED));
+ writer.AddDocument(doc);
+ doc = new Document();
+ doc.Add(new Field("partnum", "Q37", Field.Store.YES, Field.Index.TOKENIZED));
+ writer.AddDocument(doc);
+ writer.Close();
+
+ IndexReader reader = IndexReader.Open(dir);
+ TermDocs td = reader.TermDocs(new Term("partnum", "Q36"));
+ Assert.IsTrue(td.Next());
+ td = reader.TermDocs(new Term("partnum", "Q37"));
+ Assert.IsTrue(td.Next());
+ }
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestLengthFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestLengthFilter.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestLengthFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestLengthFilter.cs Tue Jul 15 14:44:04 2008
@@ -19,10 +19,13 @@
using NUnit.Framework;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
namespace Lucene.Net.Analysis
{
+
[TestFixture]
- public class TestLengthFilter
+ public class TestLengthFilter : LuceneTestCase
{
[Test]
public virtual void TestFilter()
Modified: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestPerFieldAnalzyerWrapper.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestPerFieldAnalzyerWrapper.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestPerFieldAnalzyerWrapper.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestPerFieldAnalzyerWrapper.cs Tue Jul 15 14:44:04 2008
@@ -19,12 +19,15 @@
using NUnit.Framework;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
namespace Lucene.Net.Analysis
{
- [TestFixture]
- public class TestPerFieldAnalzyerWrapper
+
+ [TestFixture]
+ public class TestPerFieldAnalzyerWrapper : LuceneTestCase
{
- [Test]
+ [Test]
public virtual void TestPerField()
{
System.String text = "Qwerty";
Modified: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStandardAnalyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestStandardAnalyzer.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStandardAnalyzer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStandardAnalyzer.cs Tue Jul 15 14:44:04 2008
@@ -20,72 +20,175 @@
using NUnit.Framework;
using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Analysis
{
+
[TestFixture]
- public class TestStandardAnalyzer
+ public class TestStandardAnalyzer : LuceneTestCase
{
+ private Analyzer a = new StandardAnalyzer();
+
public virtual void AssertAnalyzesTo(Analyzer a, System.String input, System.String[] expected)
{
+ AssertAnalyzesTo(a, input, expected, null);
+ }
+
+ public virtual void AssertAnalyzesTo(Analyzer a, System.String input, System.String[] expectedImages, System.String[] expectedTypes)
+ {
+ AssertAnalyzesTo(a, input, expectedImages, expectedTypes, null);
+ }
+
+ public virtual void AssertAnalyzesTo(Analyzer a, System.String input, System.String[] expectedImages, System.String[] expectedTypes, int[] expectedPosIncrs)
+ {
TokenStream ts = a.TokenStream("dummy", new System.IO.StringReader(input));
- for (int i = 0; i < expected.Length; i++)
+ for (int i = 0; i < expectedImages.Length; i++)
{
Token t = ts.Next();
Assert.IsNotNull(t);
- Assert.AreEqual(expected[i], t.TermText());
+ Assert.AreEqual(expectedImages[i], t.TermText());
+ if (expectedTypes != null)
+ {
+ Assert.AreEqual(expectedTypes[i], t.Type());
+ }
+ if (expectedPosIncrs != null)
+ {
+ Assert.AreEqual(expectedPosIncrs[i], t.GetPositionIncrement());
+ }
}
Assert.IsNull(ts.Next());
ts.Close();
}
+
+ [Test]
+ public virtual void TestMaxTermLength()
+ {
+ StandardAnalyzer sa = new StandardAnalyzer();
+ sa.SetMaxTokenLength(5);
+ AssertAnalyzesTo(sa, "ab cd toolong xy z", new System.String[]{"ab", "cd", "xy", "z"});
+ }
+
+ [Test]
+ public virtual void TestMaxTermLength2()
+ {
+ StandardAnalyzer sa = new StandardAnalyzer();
+ AssertAnalyzesTo(sa, "ab cd toolong xy z", new System.String[]{"ab", "cd", "toolong", "xy", "z"});
+ sa.SetMaxTokenLength(5);
+
+ AssertAnalyzesTo(sa, "ab cd toolong xy z", new System.String[]{"ab", "cd", "xy", "z"}, null, new int[]{1, 1, 2, 1});
+ }
+
[Test]
- public virtual void TestStandard()
+ public virtual void TestMaxTermLength3()
{
- Analyzer a = new StandardAnalyzer();
+ char[] chars = new char[255];
+ for (int i = 0; i < 255; i++)
+ chars[i] = 'a';
+ System.String longTerm = new System.String(chars, 0, 255);
+ AssertAnalyzesTo(a, "ab cd " + longTerm + " xy z", new System.String[]{"ab", "cd", longTerm, "xy", "z"});
+ AssertAnalyzesTo(a, "ab cd " + longTerm + "a xy z", new System.String[]{"ab", "cd", "xy", "z"});
+ }
+
+ [Test]
+ public virtual void TestAlphanumeric()
+ {
// alphanumeric tokens
AssertAnalyzesTo(a, "B2B", new System.String[]{"b2b"});
AssertAnalyzesTo(a, "2B", new System.String[]{"2b"});
-
+ }
+
+ [Test]
+ public virtual void TestUnderscores()
+ {
// underscores are delimiters, but not in email addresses (below)
AssertAnalyzesTo(a, "word_having_underscore", new System.String[]{"word", "having", "underscore"});
AssertAnalyzesTo(a, "word_with_underscore_and_stopwords", new System.String[]{"word", "underscore", "stopwords"});
-
+ }
+
+ [Test]
+ public virtual void TestDelimiters()
+ {
// other delimiters: "-", "/", ","
AssertAnalyzesTo(a, "some-dashed-phrase", new System.String[]{"some", "dashed", "phrase"});
AssertAnalyzesTo(a, "dogs,chase,cats", new System.String[]{"dogs", "chase", "cats"});
AssertAnalyzesTo(a, "ac/dc", new System.String[]{"ac", "dc"});
-
+ }
+
+ [Test]
+ public virtual void TestApostrophes()
+ {
// internal apostrophes: O'Reilly, you're, O'Reilly's
// possessives are actually removed by StardardFilter, not the tokenizer
AssertAnalyzesTo(a, "O'Reilly", new System.String[]{"o'reilly"});
AssertAnalyzesTo(a, "you're", new System.String[]{"you're"});
- AssertAnalyzesTo(a, "she's", new System.String[]{"she"});
- AssertAnalyzesTo(a, "Jim's", new System.String[]{"jim"});
- AssertAnalyzesTo(a, "don't", new System.String[]{"don't"});
- AssertAnalyzesTo(a, "O'Reilly's", new System.String[]{"o'reilly"});
-
- // t and s had been stopwords in Lucene <= 2.0, which made it impossible
- // to correctly search for these terms:
- AssertAnalyzesTo(a, "s-class", new System.String[]{"s", "class"});
- AssertAnalyzesTo(a, "t-com", new System.String[]{"t", "com"});
- // 'a' is still a stopword:
- AssertAnalyzesTo(a, "a-class", new System.String[]{"class"});
-
- // company names
+ AssertAnalyzesTo(a, "she's", new System.String[]{"she"});
+ AssertAnalyzesTo(a, "Jim's", new System.String[]{"jim"});
+ AssertAnalyzesTo(a, "don't", new System.String[]{"don't"});
+ AssertAnalyzesTo(a, "O'Reilly's", new System.String[]{"o'reilly"});
+ }
+
+ [Test]
+ public virtual void TestTSADash()
+ {
+ // t and s had been stopwords in Lucene <= 2.0, which made it impossible
+ // to correctly search for these terms:
+ AssertAnalyzesTo(a, "s-class", new System.String[]{"s", "class"});
+ AssertAnalyzesTo(a, "t-com", new System.String[]{"t", "com"});
+ // 'a' is still a stopword:
+ AssertAnalyzesTo(a, "a-class", new System.String[]{"class"});
+ }
+
+ [Test]
+ public virtual void TestCompanyNames()
+ {
+ // company names
AssertAnalyzesTo(a, "AT&T", new System.String[]{"at&t"});
AssertAnalyzesTo(a, "Excite@Home", new System.String[]{"excite@home"});
-
+ }
+
+ [Test]
+ public virtual void TestLucene1140()
+ {
+ try
+ {
+ StandardAnalyzer analyzer = new StandardAnalyzer(true);
+ AssertAnalyzesTo(analyzer, "www.nutch.org.", new System.String[]{"www.nutch.org"}, new System.String[]{"<HOST>"});
+ }
+ catch (System.NullReferenceException)
+ {
+ Assert.IsTrue(false, "Should not throw an NPE and it did");
+ }
+ }
+
+ [Test]
+ public virtual void TestDomainNames()
+ {
// domain names
AssertAnalyzesTo(a, "www.nutch.org", new System.String[]{"www.nutch.org"});
-
+ //Notice the trailing . See https://issues.apache.org/jira/browse/LUCENE-1068.
+ //TODO: Remove in 3.x
+ AssertAnalyzesTo(a, "www.nutch.org.", new System.String[]{"wwwnutchorg"}, new System.String[]{"<ACRONYM>"});
+ // the following should be recognized as HOST. The code that sets replaceDepAcronym should be removed in the next release.
+ ((StandardAnalyzer) a).SetReplaceInvalidAcronym(true);
+ AssertAnalyzesTo(a, "www.nutch.org.", new System.String[]{"www.nutch.org"}, new System.String[]{"<HOST>"});
+ }
+
+ [Test]
+ public virtual void TestEMailAddresses()
+ {
// email addresses, possibly with underscores, periods, etc
AssertAnalyzesTo(a, "test@example.com", new System.String[]{"test@example.com"});
AssertAnalyzesTo(a, "first.lastname@example.com", new System.String[]{"first.lastname@example.com"});
AssertAnalyzesTo(a, "first_lastname@example.com", new System.String[]{"first_lastname@example.com"});
-
+ }
+
+ [Test]
+ public virtual void TestNumeric()
+ {
// floating point, serial, model numbers, ip addresses, etc.
// every other segment must have at least one digit
AssertAnalyzesTo(a, "21.35", new System.String[]{"21.35"});
@@ -94,25 +197,100 @@
AssertAnalyzesTo(a, "1-2-3", new System.String[]{"1-2-3"});
AssertAnalyzesTo(a, "a1-b2-c3", new System.String[]{"a1-b2-c3"});
AssertAnalyzesTo(a, "a1-b-c3", new System.String[]{"a1-b-c3"});
-
+ }
+
+ [Test]
+ public virtual void TestTextWithNumbers()
+ {
// numbers
AssertAnalyzesTo(a, "David has 5000 bones", new System.String[]{"david", "has", "5000", "bones"});
-
+ }
+
+ [Test]
+ public virtual void TestVariousText()
+ {
// various
AssertAnalyzesTo(a, "C embedded developers wanted", new System.String[]{"c", "embedded", "developers", "wanted"});
AssertAnalyzesTo(a, "foo bar FOO BAR", new System.String[]{"foo", "bar", "foo", "bar"});
AssertAnalyzesTo(a, "foo bar . FOO <> BAR", new System.String[]{"foo", "bar", "foo", "bar"});
AssertAnalyzesTo(a, "\"QUOTED\" word", new System.String[]{"quoted", "word"});
-
+ }
+
+ [Test]
+ public virtual void TestAcronyms()
+ {
// acronyms have their dots stripped
AssertAnalyzesTo(a, "U.S.A.", new System.String[]{"usa"});
-
+ }
+
+ [Test]
+ public virtual void TestCPlusPlusHash()
+ {
// It would be nice to change the grammar in StandardTokenizer.jj to make "C#" and "C++" end up as tokens.
AssertAnalyzesTo(a, "C++", new System.String[]{"c"});
AssertAnalyzesTo(a, "C#", new System.String[]{"c"});
-
+ }
+
+ [Test]
+ public virtual void TestKorean()
+ {
// Korean words
AssertAnalyzesTo(a, "ìë
íì¸ì íê¸ì
ëë¤", new System.String[]{"ìë
íì¸ì", "íê¸ì
ëë¤"});
}
+
+ // Compliance with the "old" JavaCC-based analyzer, see:
+ // https://issues.apache.org/jira/browse/LUCENE-966#action_12516752
+
+ [Test]
+ public virtual void TestComplianceFileName()
+ {
+ AssertAnalyzesTo(a, "2004.jpg", new System.String[]{"2004.jpg"}, new System.String[]{"<HOST>"});
+ }
+
+ [Test]
+ public virtual void TestComplianceNumericIncorrect()
+ {
+ AssertAnalyzesTo(a, "62.46", new System.String[]{"62.46"}, new System.String[]{"<HOST>"});
+ }
+
+ [Test]
+ public virtual void TestComplianceNumericLong()
+ {
+ AssertAnalyzesTo(a, "978-0-94045043-1", new System.String[]{"978-0-94045043-1"}, new System.String[]{"<NUM>"});
+ }
+
+ [Test]
+ public virtual void TestComplianceNumericFile()
+ {
+ AssertAnalyzesTo(a, "78academyawards/rules/rule02.html", new System.String[]{"78academyawards/rules/rule02.html"}, new System.String[]{"<NUM>"});
+ }
+
+ [Test]
+ public virtual void TestComplianceNumericWithUnderscores()
+ {
+ AssertAnalyzesTo(a, "2006-03-11t082958z_01_ban130523_rtridst_0_ozabs", new System.String[]{"2006-03-11t082958z_01_ban130523_rtridst_0_ozabs"}, new System.String[]{"<NUM>"});
+ }
+
+ [Test]
+ public virtual void TestComplianceNumericWithDash()
+ {
+ AssertAnalyzesTo(a, "mid-20th", new System.String[]{"mid-20th"}, new System.String[]{"<NUM>"});
+ }
+
+ [Test]
+ public virtual void TestComplianceManyTokens()
+ {
+ AssertAnalyzesTo(a, "/money.cnn.com/magazines/fortune/fortune_archive/2007/03/19/8402357/index.htm " + "safari-0-sheikh-zayed-grand-mosque.jpg", new System.String[]{"money.cnn.com", "magazines", "fortune", "fortune", "archive/2007/03/19/8402357", "index.htm", "safari-0-sheikh", "zayed", "grand", "mosque.jpg"}, new System.String[]{"<HOST>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<HOST>", "<NUM>", "<ALPHANUM>", "<ALPHANUM>", "<HOST>"});
+ }
+
+ /// <deprecated> this should be removed in the 3.0.
+ /// </deprecated>
+ [Test]
+ public virtual void TestDeprecatedAcronyms()
+ {
+ // test backward compatibility for applications that require the old behavior.
+ // this should be removed once replaceDepAcronym is removed.
+ AssertAnalyzesTo(a, "lucene.apache.org.", new System.String[]{"luceneapacheorg"}, new System.String[]{"<ACRONYM>"});
+ }
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopAnalyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestStopAnalyzer.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopAnalyzer.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopAnalyzer.cs Tue Jul 15 14:44:04 2008
@@ -19,28 +19,32 @@
using NUnit.Framework;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
namespace Lucene.Net.Analysis
{
+
[TestFixture]
- public class TestStopAnalyzer
+ public class TestStopAnalyzer : LuceneTestCase
{
private StopAnalyzer stop = new StopAnalyzer();
private System.Collections.Hashtable inValidTokens = new System.Collections.Hashtable();
- [SetUp]
- public virtual void SetUp()
+ [SetUp]
+ public override void SetUp()
{
- stop = new StopAnalyzer();
- inValidTokens = new System.Collections.Hashtable();
+ base.SetUp();
+ stop = new StopAnalyzer();
+ inValidTokens = new System.Collections.Hashtable();
for (int i = 0; i < StopAnalyzer.ENGLISH_STOP_WORDS.Length; i++)
{
- inValidTokens.Add(StopAnalyzer.ENGLISH_STOP_WORDS[i], StopAnalyzer.ENGLISH_STOP_WORDS[i]);
- }
+ inValidTokens.Add(StopAnalyzer.ENGLISH_STOP_WORDS[i], StopAnalyzer.ENGLISH_STOP_WORDS[i]);
+ }
}
- [Test]
+ [Test]
public virtual void TestDefaults()
{
Assert.IsTrue(stop != null);
@@ -54,7 +58,7 @@
}
}
- [Test]
+ [Test]
public virtual void TestStopList()
{
System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
@@ -70,6 +74,38 @@
{
System.String text = token.TermText();
Assert.IsFalse(stopWordsSet.Contains(text));
+ Assert.AreEqual(1, token.GetPositionIncrement()); // by default stop tokenizer does not apply increments.
+ }
+ }
+
+ [Test]
+ public virtual void TestStopListPositions()
+ {
+ bool defaultEnable = StopFilter.GetEnablePositionIncrementsDefault();
+ StopFilter.SetEnablePositionIncrementsDefault(true);
+ try
+ {
+ System.Collections.Hashtable stopWordsSet = new System.Collections.Hashtable();
+ stopWordsSet.Add("good", "good");
+ stopWordsSet.Add("test", "test");
+ stopWordsSet.Add("analyzer", "analyzer");
+ StopAnalyzer newStop = new StopAnalyzer(stopWordsSet);
+ System.IO.StringReader reader = new System.IO.StringReader("This is a good test of the english stop analyzer with positions");
+ int[] expectedIncr = new int[]{1, 1, 1, 3, 1, 1, 1, 2, 1};
+ TokenStream stream = newStop.TokenStream("test", reader);
+ Assert.IsNotNull(stream);
+ Token token = null;
+ int i = 0;
+ while ((token = stream.Next()) != null)
+ {
+ System.String text = token.TermText();
+ Assert.IsFalse(stopWordsSet.Contains(text));
+ Assert.AreEqual(expectedIncr[i++], token.GetPositionIncrement());
+ }
+ }
+ finally
+ {
+ StopFilter.SetEnablePositionIncrementsDefault(defaultEnable);
}
}
}
Modified: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestStopFilter.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopFilter.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestStopFilter.cs Tue Jul 15 14:44:04 2008
@@ -19,16 +19,22 @@
using NUnit.Framework;
+using English = Lucene.Net.Util.English;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
namespace Lucene.Net.Analysis
{
/// <author> yonik
/// </author>
[TestFixture]
- public class TestStopFilter
+ public class TestStopFilter : LuceneTestCase
{
+ private const bool VERBOSE = false;
+
// other StopFilter functionality is already tested by TestStopAnalyzer
+
[Test]
public virtual void TestExactCase()
{
@@ -40,6 +46,7 @@
Assert.AreEqual(null, stream.Next());
}
+ [Test]
public virtual void TestIgnoreCase()
{
System.IO.StringReader reader = new System.IO.StringReader("Now is The Time");
@@ -48,5 +55,96 @@
Assert.AreEqual("Now", stream.Next().TermText());
Assert.AreEqual(null, stream.Next());
}
+
+ [Test]
+ public virtual void TestStopFilt()
+ {
+ System.IO.StringReader reader = new System.IO.StringReader("Now is The Time");
+ System.String[] stopWords = new System.String[]{"is", "the", "Time"};
+ System.Collections.Hashtable stopSet = StopFilter.MakeStopSet(stopWords);
+ TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopSet);
+ Assert.AreEqual("Now", stream.Next().TermText());
+ Assert.AreEqual("The", stream.Next().TermText());
+ Assert.AreEqual(null, stream.Next());
+ }
+
+ /// <summary> Test Position increments applied by StopFilter with and without enabling this option.</summary>
+ [Test]
+ public virtual void TestStopPositons()
+ {
+ System.Text.StringBuilder sb = new System.Text.StringBuilder();
+ System.Collections.Generic.List<string> a = new System.Collections.Generic.List<string>();
+ for (int i = 0; i < 20; i++)
+ {
+ System.String w = English.IntToEnglish(i).Trim();
+ sb.Append(w).Append(" ");
+ if (i % 3 != 0)
+ a.Add(w);
+ }
+ Log(sb.ToString());
+ System.String[] stopWords = (System.String[]) a.ToArray();
+ for (int i = 0; i < a.Count; i++)
+ Log("Stop: " + stopWords[i]);
+ System.Collections.Hashtable stopSet = StopFilter.MakeStopSet(stopWords);
+ // with increments
+ System.IO.StringReader reader = new System.IO.StringReader(sb.ToString());
+ StopFilter stpf = new StopFilter(new WhitespaceTokenizer(reader), stopSet);
+ DoTestStopPositons(stpf, true);
+ // without increments
+ reader = new System.IO.StringReader(sb.ToString());
+ stpf = new StopFilter(new WhitespaceTokenizer(reader), stopSet);
+ DoTestStopPositons(stpf, false);
+ // with increments, concatenating two stop filters
+ System.Collections.Generic.List<string> a0 = new System.Collections.Generic.List<string>();
+ System.Collections.Generic.List<string> a1 = new System.Collections.Generic.List<string>();
+ for (int i = 0; i < a.Count; i++)
+ {
+ if (i % 2 == 0)
+ {
+ a0.Add(a[i]);
+ }
+ else
+ {
+ a1.Add(a[i]);
+ }
+ }
+ System.String[] stopWords0 = (System.String[]) a0.ToArray();
+ for (int i = 0; i < a0.Count; i++)
+ Log("Stop0: " + stopWords0[i]);
+ System.String[] stopWords1 = (System.String[]) a1.ToArray();
+ for (int i = 0; i < a1.Count; i++)
+ Log("Stop1: " + stopWords1[i]);
+ System.Collections.Hashtable stopSet0 = StopFilter.MakeStopSet(stopWords0);
+ System.Collections.Hashtable stopSet1 = StopFilter.MakeStopSet(stopWords1);
+ reader = new System.IO.StringReader(sb.ToString());
+ StopFilter stpf0 = new StopFilter(new WhitespaceTokenizer(reader), stopSet0); // first part of the set
+ stpf0.SetEnablePositionIncrements(true);
+ StopFilter stpf01 = new StopFilter(stpf0, stopSet1); // two stop filters concatenated!
+ DoTestStopPositons(stpf01, true);
+ }
+
+ private void DoTestStopPositons(StopFilter stpf, bool enableIcrements)
+ {
+ Log("---> test with enable-increments-" + (enableIcrements?"enabled":"disabled"));
+ stpf.SetEnablePositionIncrements(enableIcrements);
+ for (int i = 0; i < 20; i += 3)
+ {
+ Token t = stpf.Next();
+ Log("Token " + i + ": " + t);
+ System.String w = English.IntToEnglish(i).Trim();
+ Assert.AreEqual(w, t.TermText(), "expecting token " + i + " to be " + w);
+ Assert.AreEqual(enableIcrements ? (i == 0 ? 1 : 3) : 1, t.GetPositionIncrement(), "all but first token must have position increment of 3");
+ }
+ Assert.IsNull(stpf.Next());
+ }
+
+ // print debug info depending on VERBOSE
+ private static void Log(System.String s)
+ {
+ if (VERBOSE)
+ {
+ System.Console.Out.WriteLine(s);
+ }
+ }
}
}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Test/Analysis/TestToken.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Analysis/TestToken.cs?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Analysis/TestToken.cs (added)
+++ incubator/lucene.net/trunk/C#/src/Test/Analysis/TestToken.cs Tue Jul 15 14:44:04 2008
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+using NUnit.Framework;
+
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+
+namespace Lucene.Net.Analysis
+{
+ [TestFixture]
+ public class TestToken : LuceneTestCase
+ {
+ [Test]
+ public virtual void TestToString()
+ {
+ char[] b = new char[]{'a', 'l', 'o', 'h', 'a'};
+ Token t = new Token("", 0, 5);
+ t.SetTermBuffer(b, 0, 5);
+ Assert.AreEqual("(aloha,0,5)", t.ToString());
+
+ t.SetTermText("hi there");
+ Assert.AreEqual("(hi there,0,5)", t.ToString());
+ }
+
+ [Test]
+ public virtual void TestMixedStringArray()
+ {
+ Token t = new Token("hello", 0, 5);
+ Assert.AreEqual(t.TermText(), "hello");
+ Assert.AreEqual(t.TermLength(), 5);
+ Assert.AreEqual(new System.String(t.TermBuffer(), 0, 5), "hello");
+ t.SetTermText("hello2");
+ Assert.AreEqual(t.TermLength(), 6);
+ Assert.AreEqual(new System.String(t.TermBuffer(), 0, 6), "hello2");
+ t.SetTermBuffer("hello3".ToCharArray(), 0, 6);
+ Assert.AreEqual(t.TermText(), "hello3");
+
+ // Make sure if we get the buffer and change a character
+ // that termText() reflects the change
+ char[] buffer = t.TermBuffer();
+ buffer[1] = 'o';
+ Assert.AreEqual(t.TermText(), "hollo3");
+ }
+ }
+}
\ No newline at end of file
Added: incubator/lucene.net/trunk/C#/src/Test/App.config
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/App.config?rev=677059&view=auto
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/App.config (added)
+++ incubator/lucene.net/trunk/C#/src/Test/App.config Tue Jul 15 14:44:04 2008
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<configuration>
+ <appSettings>
+ <add key="tempdir" value="c:\temp"/>
+ </appSettings>
+ <!-- when i add this setting and run tests, i get 0 success, 0 failures, 0 tests not run
+ <appSettings>
+ <add key="Lucene.Net.CompressionLib.class" value="Lucene.Net.Index.Compression.SharpZipLibAdapter"/>
+ </appSettings>
+ -->
+</configuration>
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/AssemblyInfo.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/AssemblyInfo.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/AssemblyInfo.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/AssemblyInfo.cs Tue Jul 15 14:44:04 2008
@@ -1,20 +1,3 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
using System.Reflection;
using System.Runtime.CompilerServices;
@@ -24,17 +7,16 @@
// associated with an assembly.
//
[assembly: AssemblyTitle("Apache Lucene.Net")]
-[assembly: AssemblyDescription("The Apache Software Foundation Lucene.Net full-text search library")]
+[assembly: AssemblyDescription("The Apache Software Foundation Lucene.Net a full-text search engine library")]
[assembly: AssemblyConfiguration("")]
[assembly: AssemblyCompany("The Apache Software Foundation")]
[assembly: AssemblyProduct("Lucene.Net.Test")]
-[assembly: AssemblyCopyright("Copyright 2007 The Apache Software Foundation")]
-[assembly: AssemblyTrademark("Copyright 2007 The Apache Software Foundation")]
+[assembly: AssemblyCopyright("Copyright 2006 - 2008 The Apache Software Foundation")]
+[assembly: AssemblyTrademark("Copyright 2006 - 2008 The Apache Software Foundation")]
[assembly: AssemblyDefaultAlias("Lucene.Net")]
[assembly: AssemblyCulture("")]
-[assembly: AssemblyInformationalVersionAttribute("2.1")]
-
+[assembly: AssemblyInformationalVersionAttribute("2.3.1")]
//
// Version information for an assembly consists of the following four values:
@@ -47,8 +29,7 @@
// You can specify all the values or you can default the Revision and Build Numbers
// by using the '*' as shown below:
-[assembly: AssemblyVersion("2.1.0.003")]
-
+[assembly: AssemblyVersion("2.3.1.002")]
//
// In order to sign your assembly you must specify a key to use. Refer to the
Modified: incubator/lucene.net/trunk/C#/src/Test/Document/TestBinaryDocument.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Document/TestBinaryDocument.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Document/TestBinaryDocument.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Document/TestBinaryDocument.cs Tue Jul 15 14:44:04 2008
@@ -19,32 +19,30 @@
using NUnit.Framework;
-using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
using IndexReader = Lucene.Net.Index.IndexReader;
using IndexWriter = Lucene.Net.Index.IndexWriter;
using RAMDirectory = Lucene.Net.Store.RAMDirectory;
-using Fieldable = Lucene.Net.Documents.Fieldable;
-using Field = Lucene.Net.Documents.Field;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
+using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
namespace Lucene.Net.Documents
{
/// <summary> Tests {@link Document} class.
///
+ ///
/// </summary>
- /// <author> Bernhard Messer
- /// </author>
- /// <version> $Id: TestBinaryDocument.java 387550 2006-03-21 15:36:32Z yonik $
- /// </version>
+ /// <version> $Id: TestBinaryDocument.java 598296 2007-11-26 14:52:01Z mikemccand $
+ /// </version>
[TestFixture]
- public class TestBinaryDocument
+ public class TestBinaryDocument : LuceneTestCase
{
internal System.String binaryValStored = "this text will be stored as a byte array in the index";
internal System.String binaryValCompressed = "this text will be also stored and compressed as a byte array in the index";
[Test]
- public virtual void TestBinaryFieldInIndex()
+ public virtual void TestBinaryFieldInIndex()
{
Lucene.Net.Documents.Fieldable binaryFldStored = new Field("binaryStored", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValStored), Field.Store.YES);
Lucene.Net.Documents.Fieldable binaryFldCompressed = new Field("binaryCompressed", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValCompressed), Field.Store.COMPRESS);
@@ -57,9 +55,8 @@
new Field("fail", System.Text.UTF8Encoding.UTF8.GetBytes(binaryValCompressed), Field.Store.NO);
Assert.Fail();
}
- catch (System.ArgumentException iae)
+ catch (System.ArgumentException)
{
- ;
}
Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
@@ -70,8 +67,8 @@
doc.Add(stringFldStored);
doc.Add(stringFldCompressed);
- /** test for field count */
- Assert.AreEqual(4, doc.GetFieldsCount());
+ /** test for field count */
+ Assert.AreEqual(4, doc.GetFieldsCount());
/** add the doc to a ram index */
RAMDirectory dir = new RAMDirectory();
@@ -93,11 +90,11 @@
Assert.IsTrue(binaryFldCompressedTest.Equals(binaryValCompressed));
/** fetch the string field and compare it's content with the original one */
- System.String stringFldStoredTest = new System.Text.StringBuilder(docFromReader.Get("stringStored")).ToString();
+ System.String stringFldStoredTest = docFromReader.Get("stringStored");
Assert.IsTrue(stringFldStoredTest.Equals(binaryValStored));
/** fetch the compressed string field and compare it's content with the original one */
- System.String stringFldCompressedTest = new System.Text.StringBuilder(docFromReader.Get("stringCompressed")).ToString();
+ System.String stringFldCompressedTest = docFromReader.Get("stringCompressed");
Assert.IsTrue(stringFldCompressedTest.Equals(binaryValCompressed));
/** delete the document from index */
Modified: incubator/lucene.net/trunk/C#/src/Test/Document/TestDateTools.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Document/TestDateTools.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Document/TestDateTools.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Document/TestDateTools.cs Tue Jul 15 14:44:04 2008
@@ -19,12 +19,12 @@
using NUnit.Framework;
-using DateTools = Lucene.Net.Documents.DateTools;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Documents
{
[TestFixture]
- public class TestDateTools
+ public class TestDateTools : LuceneTestCase
{
[Test]
public virtual void TestStringToDate()
@@ -45,7 +45,7 @@
d = DateTools.StringToDate("97"); // no date
Assert.Fail();
}
- catch (System.FormatException e)
+ catch (System.FormatException)
{
/* expected exception */
}
@@ -54,7 +54,7 @@
d = DateTools.StringToDate("200401011235009999"); // no date
Assert.Fail();
}
- catch (System.FormatException e)
+ catch (System.FormatException)
{
/* expected exception */
}
@@ -63,29 +63,29 @@
d = DateTools.StringToDate("aaaa"); // no date
Assert.Fail();
}
- catch (System.FormatException e)
+ catch (System.FormatException)
{
/* expected exception */
}
}
- [Test]
+ [Test]
public virtual void TestStringtoTime()
{
long time = DateTools.StringToTime("197001010000");
- System.DateTime cal = new System.DateTime(1970, 1, 1, 0, 0, 0, 0, new System.Globalization.GregorianCalendar()); // hour, minute, second
- Assert.AreEqual(cal.Ticks, time);
+ System.DateTime cal = new System.DateTime(1970, 1, 1, 0, 0, 0, 0, new System.Globalization.GregorianCalendar()); // hour, minute, second
+ Assert.AreEqual(cal.Ticks, time);
- cal = new System.DateTime(1980, 2, 2, 11, 5, 0, 0, new System.Globalization.GregorianCalendar()); // hour, minute, second
+ cal = new System.DateTime(1980, 2, 2, 11, 5, 0, 0, new System.Globalization.GregorianCalendar()); // hour, minute, second
time = DateTools.StringToTime("198002021105");
Assert.AreEqual(cal.Ticks, time);
}
- [Test]
+ [Test]
public virtual void TestDateAndTimetoString()
{
- System.DateTime cal = new System.DateTime(2004, 2, 3, 22, 8, 56, 333, new System.Globalization.GregorianCalendar());
+ System.DateTime cal = new System.DateTime(2004, 2, 3, 22, 8, 56, 333, new System.Globalization.GregorianCalendar());
System.String dateString;
System.DateTime tempAux = new System.DateTime(cal.Ticks);
@@ -131,7 +131,7 @@
Assert.AreEqual("2004-02-03 22:08:56:333", IsoFormat(tempAux14));
// date before 1970:
- cal = new System.DateTime(1961, 3, 5, 23, 9, 51, 444, new System.Globalization.GregorianCalendar());
+ cal = new System.DateTime(1961, 3, 5, 23, 9, 51, 444, new System.Globalization.GregorianCalendar());
System.DateTime tempAux15 = new System.DateTime(cal.Ticks);
dateString = DateTools.DateToString(tempAux15, DateTools.Resolution.MILLISECOND);
Assert.AreEqual("19610305230951444", dateString);
@@ -145,19 +145,19 @@
Assert.AreEqual("1961-03-05 23:00:00:000", IsoFormat(tempAux18));
// timeToString:
- cal = new System.DateTime(1970, 1, 1, 0, 0, 0, 0, new System.Globalization.GregorianCalendar());
+ cal = new System.DateTime(1970, 1, 1, 0, 0, 0, 0, new System.Globalization.GregorianCalendar());
dateString = DateTools.TimeToString(cal.Ticks, DateTools.Resolution.MILLISECOND);
Assert.AreEqual("19700101000000000", dateString);
- cal = new System.DateTime(1970, 1, 1, 1, 2, 3, 0, new System.Globalization.GregorianCalendar());
+ cal = new System.DateTime(1970, 1, 1, 1, 2, 3, 0, new System.Globalization.GregorianCalendar());
dateString = DateTools.TimeToString(cal.Ticks, DateTools.Resolution.MILLISECOND);
Assert.AreEqual("19700101010203000", dateString);
}
- [Test]
+ [Test]
public virtual void TestRound()
{
- System.DateTime date = new System.DateTime(2004, 2, 3, 22, 8, 56, 333, new System.Globalization.GregorianCalendar());
+ System.DateTime date = new System.DateTime(2004, 2, 3, 22, 8, 56, 333, new System.Globalization.GregorianCalendar());
Assert.AreEqual("2004-02-03 22:08:56:333", IsoFormat(date));
System.DateTime dateYear = DateTools.Round(date, DateTools.Resolution.YEAR);
@@ -193,31 +193,31 @@
private System.String IsoFormat(System.DateTime date)
{
- return date.ToString("yyyy-MM-dd HH:mm:ss:fff");
- }
+ return date.ToString("yyyy-MM-dd HH:mm:ss:fff");
+ }
- [Test]
- public virtual void TestDateToolsUTC()
- {
- // Sun, 30 Oct 2005 00:00:00 +0000 -- the last second of 2005's DST in Europe/London
- //long time = 1130630400;
- DateTime time1 = new DateTime(2005, 10, 30);
- DateTime time2 = time1.AddHours(1);
- try
- {
- //TimeZone.setDefault(TimeZone.getTimeZone("Europe/London")); // {{Aroush-2.0}} need porting 'java.util.TimeZone.getTimeZone'
- System.DateTime tempAux = time1;
- System.String d1 = DateTools.DateToString(tempAux, DateTools.Resolution.MINUTE);
- System.DateTime tempAux2 = time2;
- System.String d2 = DateTools.DateToString(tempAux2, DateTools.Resolution.MINUTE);
- Assert.IsFalse(d1.Equals(d2), "different times");
- Assert.AreEqual(DateTools.StringToTime(d1), time1.Ticks, "midnight");
- Assert.AreEqual(DateTools.StringToTime(d2), time2.Ticks, "later");
- }
- finally
- {
- //TimeZone.SetDefault(null); // {{Aroush-2.0}} need porting 'java.util.TimeZone.setDefault'
- }
- }
- }
+ [Test]
+ public virtual void TestDateToolsUTC()
+ {
+ // Sun, 30 Oct 2005 00:00:00 +0000 -- the last second of 2005's DST in Europe/London
+ //long time = 1130630400;
+ DateTime time1 = new DateTime(2005, 10, 30);
+ DateTime time2 = time1.AddHours(1);
+ try
+ {
+ //TimeZone.setDefault(TimeZone.getTimeZone("Europe/London")); // {{Aroush-2.0}} need porting 'java.util.TimeZone.getTimeZone'
+ System.DateTime tempAux = time1;
+ System.String d1 = DateTools.DateToString(tempAux, DateTools.Resolution.MINUTE);
+ System.DateTime tempAux2 = time2;
+ System.String d2 = DateTools.DateToString(tempAux2, DateTools.Resolution.MINUTE);
+ Assert.IsFalse(d1.Equals(d2), "different times");
+ Assert.AreEqual(DateTools.StringToTime(d1), time1.Ticks, "midnight");
+ Assert.AreEqual(DateTools.StringToTime(d2), time2.Ticks, "later");
+ }
+ finally
+ {
+ //TimeZone.SetDefault(null); // {{Aroush-2.0}} need porting 'java.util.TimeZone.setDefault'
+ }
+ }
+ }
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Document/TestDocument.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Document/TestDocument.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Document/TestDocument.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Document/TestDocument.cs Tue Jul 15 14:44:04 2008
@@ -19,36 +19,34 @@
using NUnit.Framework;
-using RAMDirectory = Lucene.Net.Store.RAMDirectory;
-using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
using IndexWriter = Lucene.Net.Index.IndexWriter;
using Term = Lucene.Net.Index.Term;
-using Query = Lucene.Net.Search.Query;
-using TermQuery = Lucene.Net.Search.TermQuery;
+using RAMDirectory = Lucene.Net.Store.RAMDirectory;
+using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
+using Hits = Lucene.Net.Search.Hits;
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+using Query = Lucene.Net.Search.Query;
using Searcher = Lucene.Net.Search.Searcher;
-using Hits = Lucene.Net.Search.Hits;
-using Fieldable = Lucene.Net.Documents.Fieldable;
-using Field = Lucene.Net.Documents.Field;
+using TermQuery = Lucene.Net.Search.TermQuery;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Documents
{
/// <summary> Tests {@link Document} class.
///
+ ///
/// </summary>
- /// <author> Otis Gospodnetic
- /// </author>
- /// <version> $Id: TestDocument.java 208846 2005-07-02 16:40:44Z dnaber $
+ /// <version> $Id: TestDocument.java 583534 2007-10-10 16:46:35Z mikemccand $
/// </version>
[TestFixture]
- public class TestDocument
+ public class TestDocument : LuceneTestCase
{
internal System.String binaryVal = "this text will be stored as a byte array in the index";
internal System.String binaryVal2 = "this text will be also stored as a byte array in the index";
- [Test]
+ [Test]
public virtual void TestBinaryField()
{
Lucene.Net.Documents.Document doc = new Lucene.Net.Documents.Document();
@@ -59,14 +57,14 @@
doc.Add(stringFld);
doc.Add(binaryFld);
- Assert.AreEqual(2, doc.GetFieldsCount());
+ Assert.AreEqual(2, doc.GetFieldsCount());
Assert.IsTrue(binaryFld.IsBinary());
Assert.IsTrue(binaryFld.IsStored());
Assert.IsFalse(binaryFld.IsIndexed());
Assert.IsFalse(binaryFld.IsTokenized());
- System.String binaryTest = (new System.Text.ASCIIEncoding()).GetString(doc.GetBinaryValue("binary"));
+ System.String binaryTest = (new System.Text.ASCIIEncoding()).GetString(doc.GetBinaryValue("binary"));
Assert.IsTrue(binaryTest.Equals(binaryVal));
System.String stringTest = doc.Get("string");
@@ -74,7 +72,7 @@
doc.Add(binaryFld2);
- Assert.AreEqual(3, doc.GetFieldsCount());
+ Assert.AreEqual(3, doc.GetFieldsCount());
byte[][] binaryTests = doc.GetBinaryValues("binary");
@@ -89,10 +87,10 @@
Assert.IsTrue(binaryTest2.Equals(binaryVal2));
doc.RemoveField("string");
- Assert.AreEqual(2, doc.GetFieldsCount());
+ Assert.AreEqual(2, doc.GetFieldsCount());
doc.RemoveFields("binary");
- Assert.AreEqual(0, doc.GetFieldsCount());
+ Assert.AreEqual(0, doc.GetFieldsCount());
}
/// <summary> Tests {@link Document#RemoveField(String)} method for a brand new Document
@@ -104,29 +102,29 @@
public virtual void TestRemoveForNewDocument()
{
Lucene.Net.Documents.Document doc = MakeDocumentWithFields();
- Assert.AreEqual(8, doc.GetFieldsCount());
+ Assert.AreEqual(8, doc.GetFieldsCount());
doc.RemoveFields("keyword");
- Assert.AreEqual(6, doc.GetFieldsCount());
+ Assert.AreEqual(6, doc.GetFieldsCount());
doc.RemoveFields("doesnotexists"); // removing non-existing fields is siltenlty ignored
doc.RemoveFields("keyword"); // removing a field more than once
- Assert.AreEqual(6, doc.GetFieldsCount());
+ Assert.AreEqual(6, doc.GetFieldsCount());
doc.RemoveField("text");
- Assert.AreEqual(5, doc.GetFieldsCount());
+ Assert.AreEqual(5, doc.GetFieldsCount());
doc.RemoveField("text");
- Assert.AreEqual(4, doc.GetFieldsCount());
+ Assert.AreEqual(4, doc.GetFieldsCount());
doc.RemoveField("text");
- Assert.AreEqual(4, doc.GetFieldsCount());
+ Assert.AreEqual(4, doc.GetFieldsCount());
doc.RemoveField("doesnotexists"); // removing non-existing fields is siltenlty ignored
- Assert.AreEqual(4, doc.GetFieldsCount());
+ Assert.AreEqual(4, doc.GetFieldsCount());
doc.RemoveFields("unindexed");
Assert.AreEqual(2, doc.GetFieldsCount());
doc.RemoveFields("unstored");
- Assert.AreEqual(0, doc.GetFieldsCount());
+ Assert.AreEqual(0, doc.GetFieldsCount());
doc.RemoveFields("doesnotexists"); // removing non-existing fields is siltenlty ignored
- Assert.AreEqual(0, doc.GetFieldsCount());
+ Assert.AreEqual(0, doc.GetFieldsCount());
}
- [Test]
+ [Test]
public virtual void TestConstructorExceptions()
{
new Field("name", "value", Field.Store.YES, Field.Index.NO); // okay
@@ -136,7 +134,7 @@
new Field("name", "value", Field.Store.NO, Field.Index.NO);
Assert.Fail();
}
- catch (System.ArgumentException e)
+ catch (System.ArgumentException)
{
// expected exception
}
@@ -146,7 +144,7 @@
new Field("name", "value", Field.Store.YES, Field.Index.NO, Field.TermVector.YES);
Assert.Fail();
}
- catch (System.ArgumentException e)
+ catch (System.ArgumentException)
{
// expected exception
}
@@ -158,7 +156,7 @@
/// </summary>
/// <throws> Exception on error </throws>
[Test]
- public virtual void TestGetValuesForNewDocument()
+ public virtual void TestGetValuesForNewDocument()
{
DoAssert(MakeDocumentWithFields(), false);
}
@@ -169,7 +167,7 @@
/// </summary>
/// <throws> Exception on error </throws>
[Test]
- public virtual void TestGetValuesForIndexedDocument()
+ public virtual void TestGetValuesForIndexedDocument()
{
RAMDirectory dir = new RAMDirectory();
IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true);
@@ -234,5 +232,49 @@
Assert.IsTrue(unstoredFieldValues[1].Equals("test2"));
}
}
+
+ [Test]
+ public virtual void TestFieldSetValue()
+ {
+
+ Field field = new Field("id", "id1", Field.Store.YES, Field.Index.UN_TOKENIZED);
+ Document doc = new Document();
+ doc.Add(field);
+ doc.Add(new Field("keyword", "test", Field.Store.YES, Field.Index.UN_TOKENIZED));
+
+ RAMDirectory dir = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true);
+ writer.AddDocument(doc);
+ field.SetValue("id2");
+ writer.AddDocument(doc);
+ field.SetValue("id3");
+ writer.AddDocument(doc);
+ writer.Close();
+
+ Searcher searcher = new IndexSearcher(dir);
+
+ Query query = new TermQuery(new Term("keyword", "test"));
+
+ // ensure that queries return expected results without DateFilter first
+ Hits hits = searcher.Search(query);
+ Assert.AreEqual(3, hits.Length());
+ int result = 0;
+ for (int i = 0; i < 3; i++)
+ {
+ Document doc2 = hits.Doc(i);
+ Field f = doc2.GetField("id");
+ if (f.StringValue().Equals("id1"))
+ result |= 1;
+ else if (f.StringValue().Equals("id2"))
+ result |= 2;
+ else if (f.StringValue().Equals("id3"))
+ result |= 4;
+ else
+ Assert.Fail("unexpected id field");
+ }
+ searcher.Close();
+ dir.Close();
+ Assert.AreEqual(7, result, "did not see all IDs");
+ }
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/C#/src/Test/Document/TestNumberTools.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/C%23/src/Test/Document/TestNumberTools.cs?rev=677059&r1=677058&r2=677059&view=diff
==============================================================================
--- incubator/lucene.net/trunk/C#/src/Test/Document/TestNumberTools.cs (original)
+++ incubator/lucene.net/trunk/C#/src/Test/Document/TestNumberTools.cs Tue Jul 15 14:44:04 2008
@@ -19,14 +19,14 @@
using NUnit.Framework;
-using NumberTools = Lucene.Net.Documents.NumberTools;
+using LuceneTestCase = Lucene.Net.Util.LuceneTestCase;
namespace Lucene.Net.Documents
{
[TestFixture]
- public class TestNumberTools
+ public class TestNumberTools : LuceneTestCase
{
- [Test]
+ [Test]
public virtual void TestNearZero()
{
for (int i = - 100; i <= 100; i++)
@@ -38,7 +38,7 @@
}
}
- [Test]
+ [Test]
public virtual void TestMax()
{
// make sure the constants convert to their equivelents
@@ -52,7 +52,7 @@
}
}
- [Test]
+ [Test]
public virtual void TestMin()
{
// make sure the constants convert to their equivelents