You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by cc...@apache.org on 2011/11/21 05:47:21 UTC
[Lucene.Net] svn commit: r1204354 - in
/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers:
./ AR/ NGram/ Shingle/
Author: ccurrens
Date: Mon Nov 21 04:47:20 2011
New Revision: 1204354
URL: http://svn.apache.org/viewvc?rev=1204354&view=rev
Log:
missing tests from last commit that build
Added:
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/PortedTests.txt
Modified:
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/AR/TestArabicAnalyzer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenFilter.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenizer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleAnalyzerWrapperTest.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleFilterTest.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/TestShingleMatrixFilter.cs
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/AR/TestArabicAnalyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/AR/TestArabicAnalyzer.cs?rev=1204354&r1=1204353&r2=1204354&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/AR/TestArabicAnalyzer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/AR/TestArabicAnalyzer.cs Mon Nov 21 04:47:20 2011
@@ -23,6 +23,7 @@ using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Tokenattributes;
using Lucene.Net.Util;
using NUnit.Framework;
+using Version = Lucene.Net.Util.Version;
namespace Lucene.Net.Analysis.AR
{
@@ -41,7 +42,7 @@ namespace Lucene.Net.Analysis.AR
[Test]
public void TestResourcesAvailable()
{
- new ArabicAnalyzer();
+ new ArabicAnalyzer(Version.LUCENE_CURRENT);
}
/**
@@ -50,7 +51,7 @@ namespace Lucene.Net.Analysis.AR
[Test]
public void TestBasicFeatures()
{
- ArabicAnalyzer a = new ArabicAnalyzer();
+ ArabicAnalyzer a = new ArabicAnalyzer(Version.LUCENE_CURRENT);
AssertAnalyzesTo(a, "ÙبÙر", new String[] { "ÙبÙر" });
AssertAnalyzesTo(a, "ÙبÙرة", new String[] { "ÙبÙر" }); // feminine marker
@@ -73,7 +74,7 @@ namespace Lucene.Net.Analysis.AR
[Test]
public void TestReusableTokenStream()
{
- ArabicAnalyzer a = new ArabicAnalyzer();
+ ArabicAnalyzer a = new ArabicAnalyzer(Version.LUCENE_CURRENT);
AssertAnalyzesToReuse(a, "ÙبÙر", new String[] { "ÙبÙر" });
AssertAnalyzesToReuse(a, "ÙبÙرة", new String[] { "ÙبÙر" }); // feminine marker
}
@@ -84,7 +85,7 @@ namespace Lucene.Net.Analysis.AR
[Test]
public void TestEnglishInput()
{
- AssertAnalyzesTo(new ArabicAnalyzer(), "English text.", new String[] {
+ AssertAnalyzesTo(new ArabicAnalyzer(Version.LUCENE_CURRENT), "English text.", new String[] {
"english", "text" });
}
@@ -94,7 +95,7 @@ namespace Lucene.Net.Analysis.AR
[Test]
public void TestCustomStopwords()
{
- ArabicAnalyzer a = new ArabicAnalyzer(new String[] { "the", "and", "a" });
+ ArabicAnalyzer a = new ArabicAnalyzer(Version.LUCENE_CURRENT, new String[] { "the", "and", "a" });
AssertAnalyzesTo(a, "The quick brown fox.", new String[] { "quick", "brown", "fox" });
}
}
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenFilter.cs?rev=1204354&r1=1204353&r2=1204354&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenFilter.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenFilter.cs Mon Nov 21 04:47:20 2011
@@ -48,7 +48,7 @@ namespace Lucene.Net.Analysis.NGram
bool gotException = false;
try
{
- new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 0, 0);
+ new EdgeNGramTokenFilter(input, Side.FRONT, 0, 0);
}
catch (System.ArgumentException e)
{
@@ -63,7 +63,7 @@ namespace Lucene.Net.Analysis.NGram
bool gotException = false;
try
{
- new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 2, 1);
+ new EdgeNGramTokenFilter(input, Side.FRONT, 2, 1);
}
catch (System.ArgumentException e)
{
@@ -78,7 +78,7 @@ namespace Lucene.Net.Analysis.NGram
bool gotException = false;
try
{
- new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, -1, 2);
+ new EdgeNGramTokenFilter(input, Side.FRONT, -1, 2);
}
catch (System.ArgumentException e)
{
@@ -90,35 +90,35 @@ namespace Lucene.Net.Analysis.NGram
[Test]
public void TestFrontUnigram()
{
- EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 1, 1);
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, Side.FRONT, 1, 1);
AssertTokenStreamContents(tokenizer, new String[] { "a" }, new int[] { 0 }, new int[] { 1 });
}
[Test]
public void TestBackUnigram()
{
- EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.BACK, 1, 1);
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, Side.BACK, 1, 1);
AssertTokenStreamContents(tokenizer, new String[] { "e" }, new int[] { 4 }, new int[] { 5 });
}
[Test]
public void TestOversizedNgrams()
{
- EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 6, 6);
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, Side.FRONT, 6, 6);
AssertTokenStreamContents(tokenizer, new String[0], new int[0], new int[0]);
}
[Test]
public void TestFrontRangeOfNgrams()
{
- EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, Side.FRONT, 1, 3);
AssertTokenStreamContents(tokenizer, new String[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 });
}
[Test]
public void TestBackRangeOfNgrams()
{
- EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.BACK, 1, 3);
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, Side.BACK, 1, 3);
AssertTokenStreamContents(tokenizer, new String[] { "e", "de", "cde" }, new int[] { 4, 3, 2 }, new int[] { 5, 5, 5 });
}
@@ -126,7 +126,7 @@ namespace Lucene.Net.Analysis.NGram
public void TestSmallTokenInStream()
{
input = new WhitespaceTokenizer(new StringReader("abc de fgh"));
- EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, EdgeNGramTokenFilter.Side.FRONT, 3, 3);
+ EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(input, Side.FRONT, 3, 3);
AssertTokenStreamContents(tokenizer, new String[] { "abc", "fgh" }, new int[] { 0, 7 }, new int[] { 3, 10 });
}
@@ -134,7 +134,7 @@ namespace Lucene.Net.Analysis.NGram
public void TestReset()
{
WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(new StringReader("abcde"));
- EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(tokenizer, EdgeNGramTokenFilter.Side.FRONT, 1, 3);
+ EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(tokenizer, Side.FRONT, 1, 3);
AssertTokenStreamContents(filter, new String[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 });
tokenizer.Reset(new StringReader("abcde"));
AssertTokenStreamContents(filter, new String[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 });
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenizer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenizer.cs?rev=1204354&r1=1204353&r2=1204354&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenizer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/NGram/TestEdgeNGramTokenizer.cs Mon Nov 21 04:47:20 2011
@@ -48,7 +48,7 @@ namespace Lucene.Net.Analysis.NGram
bool gotException = false;
try
{
- new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 0, 0);
+ new EdgeNGramTokenizer(input, Side.FRONT, 0, 0);
}
catch (System.ArgumentException e)
{
@@ -63,7 +63,7 @@ namespace Lucene.Net.Analysis.NGram
bool gotException = false;
try
{
- new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 2, 1);
+ new EdgeNGramTokenizer(input, Side.FRONT, 2, 1);
}
catch (System.ArgumentException e)
{
@@ -78,7 +78,7 @@ namespace Lucene.Net.Analysis.NGram
bool gotException = false;
try
{
- new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, -1, 2);
+ new EdgeNGramTokenizer(input, Side.FRONT, -1, 2);
}
catch (System.ArgumentException e)
{
@@ -90,42 +90,42 @@ namespace Lucene.Net.Analysis.NGram
[Test]
public void TestFrontUnigram()
{
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 1, 1);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.FRONT, 1, 1);
AssertTokenStreamContents(tokenizer, new String[] { "a" }, new int[] { 0 }, new int[] { 1 }, 5 /* abcde */);
}
[Test]
public void TestBackUnigram()
{
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.BACK, 1, 1);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.BACK, 1, 1);
AssertTokenStreamContents(tokenizer, new String[] { "e" }, new int[] { 4 }, new int[] { 5 }, 5 /* abcde */);
}
[Test]
public void TestOversizedNgrams()
{
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 6, 6);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.FRONT, 6, 6);
AssertTokenStreamContents(tokenizer, new String[0], new int[0], new int[0], 5 /* abcde */);
}
[Test]
public void TestFrontRangeOfNgrams()
{
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 1, 3);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.FRONT, 1, 3);
AssertTokenStreamContents(tokenizer, new String[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, 5 /* abcde */);
}
[Test]
public void TestBackRangeOfNgrams()
{
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.BACK, 1, 3);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.BACK, 1, 3);
AssertTokenStreamContents(tokenizer, new String[] { "e", "de", "cde" }, new int[] { 4, 3, 2 }, new int[] { 5, 5, 5 }, 5 /* abcde */);
}
[Test]
public void TestReset()
{
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, EdgeNGramTokenizer.Side.FRONT, 1, 3);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(input, Side.FRONT, 1, 3);
AssertTokenStreamContents(tokenizer, new String[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, 5 /* abcde */);
tokenizer.Reset(new StringReader("abcde"));
AssertTokenStreamContents(tokenizer, new String[] { "a", "ab", "abc" }, new int[] { 0, 0, 0 }, new int[] { 1, 2, 3 }, 5 /* abcde */);
Added: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/PortedTests.txt
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/PortedTests.txt?rev=1204354&view=auto
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/PortedTests.txt (added)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/PortedTests.txt Mon Nov 21 04:47:20 2011
@@ -0,0 +1,48 @@
+
+analysis\ar\TestArabicAnalyzer.java - Text files are different
+analysis\ar\TestArabicNormalizationFilter.java - Text files are identical
+analysis\ar\TestArabicStemFilter.java - Text files are identical
+analysis\br\TestBrazilianStemmer.java - Text files are different
+analysis\cjk\TestCJKTokenizer.java - Text files are different
+analysis\cn\TestChineseTokenizer.java - Text files are different
+analysis\compound\TestCompoundWordTokenFilter.java - Text files are different
+analysis\cz\customStopWordFile.txt - Text files are identical
+analysis\cz\TestCzechAnalyzer.java - Text files are different
+analysis\de\data.txt - Text files are identical
+analysis\de\TestGermanStemFilter.java - Text files are different
+analysis\el\GreekAnalyzerTest.java - Text files are different
+analysis\fa\TestPersianAnalyzer.java - Text files are different
+analysis\fa\TestPersianNormalizationFilter.java - Text files are identical
+analysis\fr\TestElision.java - Text files are different
+analysis\fr\TestFrenchAnalyzer.java - Text files are different
+analysis\miscellaneous\PatternAnalyzerTest.java - Right only: C:\SVN\apache\Lucene\lucene-3.0.3\contrib\analyzers\common\src\test\org\apache\lucene\analysis\miscellaneous
+analysis\miscellaneous\TestEmptyTokenStream.java - Text files are identical
+analysis\miscellaneous\TestPrefixAndSuffixAwareTokenFilter.java - Text files are identical
+analysis\miscellaneous\TestPrefixAwareTokenFilter.java - Text files are identical
+analysis\miscellaneous\TestSingleTokenTokenFilter.java - Text files are different
+analysis\ngram\EdgeNGramTokenFilterTest.java - Text files are different
+analysis\ngram\EdgeNGramTokenizerTest.java - Text files are different
+analysis\ngram\NGramTokenFilterTest.java - Text files are different
+analysis\ngram\NGramTokenizerTest.java - Text files are different
+analysis\nl\customStemDict.txt - Text files are identical
+analysis\nl\TestDutchStemmer.java - Text files are different
+analysis\payloads\DelimitedPayloadTokenFilterTest.java - Text files are different
+analysis\payloads\NumericPayloadTokenFilterTest.java - Text files are different
+analysis\payloads\TokenOffsetPayloadTokenFilterTest.java - Text files are different
+analysis\payloads\TypeAsPayloadTokenFilterTest.java - Text files are different
+analysis\position\PositionFilterTest.java - Text files are different
+analysis\query\QueryAutoStopWordAnalyzerTest.java - Text files are different
+analysis\reverse\TestReverseStringFilter.java - Text files are different
+analysis\ru\resUTF8.htm - Text files are identical
+analysis\ru\stemsUTF8.txt - Text files are identical
+analysis\ru\TestRussianAnalyzer.java - Text files are different
+analysis\ru\TestRussianStem.java - Text files are different
+analysis\ru\testUTF8.txt - Text files are identical
+analysis\ru\wordsUTF8.txt - Text files are identical
+analysis\shingle\ShingleAnalyzerWrapperTest.java - Text files are different
+analysis\shingle\ShingleFilterTest.java - Text files are different
+analysis\shingle\TestShingleMatrixFilter.java - Text files are different
+analysis\sinks\DateRecognizerSinkTokenizerTest.java - Text files are identical
+analysis\sinks\TokenRangeSinkTokenizerTest.java - Text files are identical
+analysis\sinks\TokenTypeSinkTokenizerTest.java - Text files are different
+analysis\th\TestThaiAnalyzer.java - Text files are different
\ No newline at end of file
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleAnalyzerWrapperTest.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleAnalyzerWrapperTest.cs?rev=1204354&r1=1204353&r2=1204354&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleAnalyzerWrapperTest.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleAnalyzerWrapperTest.cs Mon Nov 21 04:47:20 2011
@@ -44,7 +44,7 @@ namespace Lucene.Net.Analyzers.Shingle
public IndexSearcher SetUpSearcher(Analyzer analyzer)
{
Directory dir = new RAMDirectory();
- var writer = new IndexWriter(dir, analyzer, true);
+ var writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.UNLIMITED);
var doc = new Document();
doc.Add(new Field("content", "please divide this sentence into shingles",
@@ -63,26 +63,26 @@ namespace Lucene.Net.Analyzers.Shingle
writer.Close();
- return new IndexSearcher(dir);
+ return new IndexSearcher(dir, true);
}
- protected Hits QueryParsingTest(Analyzer analyzer, String qs)
+ protected ScoreDoc[] QueryParsingTest(Analyzer analyzer, String qs)
{
Searcher = SetUpSearcher(analyzer);
- var qp = new QueryParser("content", analyzer);
+ var qp = new QueryParser(Util.Version.LUCENE_CURRENT, "content", analyzer);
var q = qp.Parse(qs);
- return Searcher.Search(q);
+ return Searcher.Search(q, null, 1000).ScoreDocs;
}
- protected void CompareRanks(Hits hits, int[] ranks)
+ protected void CompareRanks(ScoreDoc[] hits, int[] ranks)
{
- Assert.AreEqual(ranks.Length, hits.Length());
+ Assert.AreEqual(ranks.Length, hits.Length);
for (int i = 0; i < ranks.Length; i++)
{
- Assert.AreEqual(ranks[i], hits.Id(i));
+ Assert.AreEqual(ranks[i], hits[i].doc);
}
}
@@ -148,8 +148,8 @@ namespace Lucene.Net.Analyzers.Shingle
var ts = analyzer.TokenStream("content", new StringReader("this sentence"));
var j = -1;
- var posIncrAtt = (PositionIncrementAttribute) ts.AddAttribute(typeof (PositionIncrementAttribute));
- var termAtt = (TermAttribute) ts.AddAttribute(typeof (TermAttribute));
+ var posIncrAtt = ts.AddAttribute<PositionIncrementAttribute>();
+ var termAtt = ts.AddAttribute<TermAttribute>();
while (ts.IncrementToken())
{
@@ -158,7 +158,7 @@ namespace Lucene.Net.Analyzers.Shingle
q.Add(new Term("content", termText), j);
}
- var hits = Searcher.Search(q);
+ var hits = Searcher.Search(q, null, 1000).ScoreDocs;
var ranks = new[] {0};
CompareRanks(hits, ranks);
}
@@ -178,7 +178,7 @@ namespace Lucene.Net.Analyzers.Shingle
var ts = analyzer.TokenStream("content", new StringReader("test sentence"));
- var termAtt = (TermAttribute) ts.AddAttribute(typeof (TermAttribute));
+ var termAtt = ts.AddAttribute<TermAttribute>();
while (ts.IncrementToken())
{
@@ -187,7 +187,7 @@ namespace Lucene.Net.Analyzers.Shingle
BooleanClause.Occur.SHOULD);
}
- var hits = Searcher.Search(q);
+ var hits = Searcher.Search(q, null, 1000).ScoreDocs;
var ranks = new[] {1, 2, 0};
CompareRanks(hits, ranks);
}
@@ -220,45 +220,9 @@ namespace Lucene.Net.Analyzers.Shingle
{
Analyzer a = new ShingleWrapperSubclassAnalyzer();
AssertAnalyzesToReuse(a, "this is a test",
- new[] {"this", "is", "a", "test"},
- new[] {0, 5, 8, 10},
- new[] {4, 7, 9, 14});
- }
-
- /// <summary>
- /// analyzer that does not support reuse it is LetterTokenizer on odd invocations, WhitespaceTokenizer on even.
- /// </summary>
- [Test]
- public void TestWrappedAnalyzerDoesNotReuse()
- {
- Analyzer a = new ShingleAnalyzerWrapper(new NonreusableAnalyzer());
- AssertAnalyzesToReuse(a, "please divide into shingles.",
- new[]
- {
- "please", "please divide", "divide", "divide into", "into", "into shingles",
- "shingles"
- },
- new[] {0, 0, 7, 7, 14, 14, 19},
- new[] {6, 13, 13, 18, 18, 27, 27},
- new[] {1, 0, 1, 0, 1, 0, 1});
- AssertAnalyzesToReuse(a, "please divide into shingles.",
- new[]
- {
- "please", "please divide", "divide", "divide into", "into", "into shingles.",
- "shingles."
- },
- new[] {0, 0, 7, 7, 14, 14, 19},
- new[] {6, 13, 13, 18, 18, 28, 28},
- new[] {1, 0, 1, 0, 1, 0, 1});
- AssertAnalyzesToReuse(a, "please divide into shingles.",
- new[]
- {
- "please", "please divide", "divide", "divide into", "into", "into shingles",
- "shingles"
- },
- new[] {0, 0, 7, 7, 14, 14, 19},
- new[] {6, 13, 13, 18, 18, 27, 27},
- new[] {1, 0, 1, 0, 1, 0, 1});
+ new[] { "this", "is", "a", "test" },
+ new[] { 0, 5, 8, 10 },
+ new[] { 4, 7, 9, 14 });
}
#region Nested type: NonreusableAnalyzer
@@ -282,6 +246,12 @@ namespace Lucene.Net.Analyzers.Shingle
private class ShingleWrapperSubclassAnalyzer : ShingleAnalyzerWrapper
{
+ public ShingleWrapperSubclassAnalyzer()
+ : base(Util.Version.LUCENE_CURRENT)
+ {
+
+ }
+
public override TokenStream TokenStream(String fieldName, TextReader reader)
{
return new WhitespaceTokenizer(reader);
@@ -289,5 +259,41 @@ namespace Lucene.Net.Analyzers.Shingle
} ;
#endregion
+
+ /// <summary>
+ /// analyzer that does not support reuse it is LetterTokenizer on odd invocations, WhitespaceTokenizer on even.
+ /// </summary>
+ [Test]
+ public void TestWrappedAnalyzerDoesNotReuse()
+ {
+ Analyzer a = new ShingleAnalyzerWrapper(new NonreusableAnalyzer());
+ AssertAnalyzesToReuse(a, "please divide into shingles.",
+ new[]
+ {
+ "please", "please divide", "divide", "divide into", "into", "into shingles",
+ "shingles"
+ },
+ new[] { 0, 0, 7, 7, 14, 14, 19 },
+ new[] { 6, 13, 13, 18, 18, 27, 27 },
+ new[] { 1, 0, 1, 0, 1, 0, 1 });
+ AssertAnalyzesToReuse(a, "please divide into shingles.",
+ new[]
+ {
+ "please", "please divide", "divide", "divide into", "into", "into shingles.",
+ "shingles."
+ },
+ new[] { 0, 0, 7, 7, 14, 14, 19 },
+ new[] { 6, 13, 13, 18, 18, 28, 28 },
+ new[] { 1, 0, 1, 0, 1, 0, 1 });
+ AssertAnalyzesToReuse(a, "please divide into shingles.",
+ new[]
+ {
+ "please", "please divide", "divide", "divide into", "into", "into shingles",
+ "shingles"
+ },
+ new[] { 0, 0, 7, 7, 14, 14, 19 },
+ new[] { 6, 13, 13, 18, 18, 27, 27 },
+ new[] { 1, 0, 1, 0, 1, 0, 1 });
+ }
}
}
\ No newline at end of file
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleFilterTest.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleFilterTest.cs?rev=1204354&r1=1204353&r2=1204354&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleFilterTest.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/ShingleFilterTest.cs Mon Nov 21 04:47:20 2011
@@ -456,10 +456,10 @@ namespace Lucene.Net.Analyzers.Shingle
var filter = new ShingleFilter(new TestTokenStream(tokensToShingle), maxSize);
filter.SetOutputUnigrams(outputUnigrams);
- var termAtt = (TermAttribute) filter.AddAttribute(typeof (TermAttribute));
- var offsetAtt = (OffsetAttribute) filter.AddAttribute(typeof (OffsetAttribute));
- var posIncrAtt = (PositionIncrementAttribute) filter.AddAttribute(typeof (PositionIncrementAttribute));
- var typeAtt = (TypeAttribute) filter.AddAttribute(typeof (TypeAttribute));
+ var termAtt = filter.AddAttribute<TermAttribute>();
+ var offsetAtt = filter.AddAttribute<OffsetAttribute>();
+ var posIncrAtt = filter.AddAttribute<PositionIncrementAttribute>();
+ var typeAtt = filter.AddAttribute<TypeAttribute>();
int i = 0;
while (filter.IncrementToken())
@@ -501,10 +501,10 @@ namespace Lucene.Net.Analyzers.Shingle
{
_testToken = testToken;
- _termAtt = (TermAttribute) AddAttribute(typeof (TermAttribute));
- _offsetAtt = (OffsetAttribute) AddAttribute(typeof (OffsetAttribute));
- _posIncrAtt = (PositionIncrementAttribute) AddAttribute(typeof (PositionIncrementAttribute));
- _typeAtt = (TypeAttribute) AddAttribute(typeof (TypeAttribute));
+ _termAtt = AddAttribute<TermAttribute>();
+ _offsetAtt = AddAttribute<OffsetAttribute>();
+ _posIncrAtt = AddAttribute<PositionIncrementAttribute>();
+ _typeAtt = AddAttribute<TypeAttribute>();
}
public override bool IncrementToken()
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/TestShingleMatrixFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/TestShingleMatrixFilter.cs?rev=1204354&r1=1204353&r2=1204354&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/TestShingleMatrixFilter.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Analyzers/Shingle/TestShingleMatrixFilter.cs Mon Nov 21 04:47:20 2011
@@ -20,11 +20,11 @@ using System.Collections;
using System.Collections.Generic;
using System.IO;
using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Shingle.Codec;
+using Lucene.Net.Analysis.Shingle.Matrix;
using Lucene.Net.Analysis.Tokenattributes;
using Lucene.Net.Analyzers.Miscellaneous;
using Lucene.Net.Analyzers.Payloads;
-using Lucene.Net.Analyzers.Shingle.Codec;
-using Lucene.Net.Analyzers.Shingle.Matrix;
using NUnit.Framework;
using FlagsAttribute = Lucene.Net.Analysis.Tokenattributes.FlagsAttribute;
@@ -38,22 +38,7 @@ namespace Lucene.Net.Analyzers.Shingle
// use this ctor, because SingleTokenTokenStream only uses next(Token), so exclude it
public TestShingleMatrixFilter(String name) :
- base(
- name,
- new Hashtable(
- new Dictionary<string, string[]>
- {
- {
- "TestShingleMatrixFilter",
- new[]
- {
- "testBehavingAsShingleFilter",
- "testMatrix",
- "testIterator"
- }
- }
- }
- ))
+ base(name)
{
}
@@ -373,7 +358,7 @@ namespace Lucene.Net.Analyzers.Shingle
// set it here in case tests are run out of the usual order.
ShingleMatrixFilter.DefaultSettingsCodec = new SimpleThreeDimensionalTokenSettingsCodec();
- var matrix = new Matrix.Matrix();
+ var matrix = new Matrix();
new Column(TokenFactory("no", 1), matrix);
new Column(TokenFactory("surprise", 1), matrix);
@@ -447,44 +432,45 @@ namespace Lucene.Net.Analyzers.Shingle
Assert.IsFalse(ts.IncrementToken());
}
- private static Token TokenFactory(String text, int posIncr, int startOffset, int endOffset)
+ private Token TokenFactory(String text, int startOffset, int endOffset)
{
- var token = new Token(startOffset, endOffset);
+ return TokenFactory(text, 1, 1f, startOffset, endOffset);
+ }
+ private Token TokenFactory(String text, int posIncr, int startOffset, int endOffset)
+ {
+ Token token = new Token(startOffset, endOffset);
token.SetTermBuffer(text);
token.SetPositionIncrement(posIncr);
-
return token;
}
-
- private static Token TokenFactory(String text, int posIncr)
+
+ private Token TokenFactory(String text, int posIncr)
{
return TokenFactory(text, posIncr, 1f, 0, 0);
}
- private static Token TokenFactory(String text, int posIncr, float weight, int startOffset, int endOffset)
+ private Token TokenFactory(String text, int posIncr, float weight)
{
- var token = new Token(startOffset, endOffset);
+ return TokenFactory(text, posIncr, weight, 0, 0);
+ }
+ private Token TokenFactory(String text, int posIncr, float weight, int startOffset, int endOffset)
+ {
+ Token token = new Token(startOffset, endOffset);
token.SetTermBuffer(text);
token.SetPositionIncrement(posIncr);
-
ShingleMatrixFilter.DefaultSettingsCodec.SetWeight(token, weight);
-
return token;
}
- private static Token TokenFactory(String text, int posIncr, float weight, int startOffset, int endOffset,
- TokenPositioner positioner)
+ private Token TokenFactory(String text, int posIncr, float weight, int startOffset, int endOffset, TokenPositioner positioner)
{
- var token = new Token(startOffset, endOffset);
-
+ Token token = new Token(startOffset, endOffset);
token.SetTermBuffer(text);
token.SetPositionIncrement(posIncr);
-
ShingleMatrixFilter.DefaultSettingsCodec.SetWeight(token, weight);
ShingleMatrixFilter.DefaultSettingsCodec.SetTokenPositioner(token, positioner);
-
return token;
}
@@ -492,7 +478,7 @@ namespace Lucene.Net.Analyzers.Shingle
private static void AssertNext(TokenStream ts, String text)
{
- var termAtt = (TermAttribute) ts.AddAttribute(typeof (TermAttribute));
+ var termAtt = ts.AddAttribute<TermAttribute>();
Assert.IsTrue(ts.IncrementToken());
Assert.AreEqual(text, termAtt.Term());
@@ -501,10 +487,10 @@ namespace Lucene.Net.Analyzers.Shingle
private static void AssertNext(TokenStream ts, String text, int positionIncrement, float boost, int startOffset,
int endOffset)
{
- var termAtt = (TermAttribute) ts.AddAttribute(typeof (TermAttribute));
- var posIncrAtt = (PositionIncrementAttribute) ts.AddAttribute(typeof (PositionIncrementAttribute));
- var payloadAtt = (PayloadAttribute) ts.AddAttribute(typeof (PayloadAttribute));
- var offsetAtt = (OffsetAttribute) ts.AddAttribute(typeof (OffsetAttribute));
+ var termAtt = ts.AddAttribute<TermAttribute>();
+ var posIncrAtt = ts.AddAttribute<PositionIncrementAttribute>();
+ var payloadAtt = ts.AddAttribute<PayloadAttribute>();
+ var offsetAtt = ts.AddAttribute<OffsetAttribute>();
Assert.IsTrue(ts.IncrementToken());
Assert.AreEqual(text, termAtt.Term());
@@ -519,8 +505,8 @@ namespace Lucene.Net.Analyzers.Shingle
private static void AssertNext(TokenStream ts, String text, int startOffset, int endOffset)
{
- var termAtt = (TermAttribute) ts.AddAttribute(typeof (TermAttribute));
- var offsetAtt = (OffsetAttribute) ts.AddAttribute(typeof (OffsetAttribute));
+ var termAtt = ts.AddAttribute<TermAttribute>();
+ var offsetAtt = ts.AddAttribute<OffsetAttribute>();
Assert.IsTrue(ts.IncrementToken());
Assert.AreEqual(text, termAtt.Term());
@@ -552,12 +538,12 @@ namespace Lucene.Net.Analyzers.Shingle
public TokenListStream(ICollection<Token> tokens)
{
_tokens = tokens;
- _termAtt = (TermAttribute) AddAttribute(typeof (TermAttribute));
- _posIncrAtt = (PositionIncrementAttribute) AddAttribute(typeof (PositionIncrementAttribute));
- _payloadAtt = (PayloadAttribute) AddAttribute(typeof (PayloadAttribute));
- _offsetAtt = (OffsetAttribute) AddAttribute(typeof (OffsetAttribute));
- _typeAtt = (TypeAttribute) AddAttribute(typeof (TypeAttribute));
- _flagsAtt = (FlagsAttribute) AddAttribute(typeof (FlagsAttribute));
+ _termAtt = AddAttribute<TermAttribute>();
+ _posIncrAtt = AddAttribute<PositionIncrementAttribute>();
+ _payloadAtt = AddAttribute<PayloadAttribute>();
+ _offsetAtt = AddAttribute<OffsetAttribute>();
+ _typeAtt = AddAttribute<TypeAttribute>();
+ _flagsAtt = AddAttribute<FlagsAttribute>();
}
public override bool IncrementToken()