You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by cc...@apache.org on 2012/02/28 23:43:28 UTC
[Lucene.Net] svn commit: r1294875 [8/45] - in /incubator/lucene.net/trunk: ./ build/
build/vs2010/contrib/ build/vs2010/test/ doc/ src/ src/contrib/Analyzers/
src/contrib/Analyzers/AR/ src/contrib/Analyzers/BR/
src/contrib/Analyzers/CJK/ src/contrib/Analyzers/Cn/ s...
Modified: incubator/lucene.net/trunk/src/contrib/Queries/FuzzyLikeThisQuery.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Queries/FuzzyLikeThisQuery.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Queries/FuzzyLikeThisQuery.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Queries/FuzzyLikeThisQuery.cs Tue Feb 28 22:43:08 2012
@@ -25,6 +25,7 @@ using Lucene.Net.Search;
using Lucene.Net.Index;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Support;
using Lucene.Net.Util;
namespace Lucene.Net.Search
@@ -48,7 +49,7 @@ namespace Lucene.Net.Search
{
static Similarity sim = new DefaultSimilarity();
Query rewrittenQuery = null;
- ArrayList fieldVals = new ArrayList();
+ EquatableList<FieldVals> fieldVals = new EquatableList<FieldVals>();
Analyzer analyzer;
ScoreTermQueue q;
@@ -89,7 +90,7 @@ namespace Lucene.Net.Search
if (other.fieldVals != null)
return false;
}
- else if (!fieldVals.EqualsToArrayList(other.fieldVals))
+ else if (!fieldVals.Equals(other.fieldVals))
return false;
if (ignoreTF != other.ignoreTF)
return false;
@@ -190,17 +191,17 @@ namespace Lucene.Net.Search
{
if (f.queryString == null) return;
TokenStream ts = analyzer.TokenStream(f.fieldName, new System.IO.StringReader(f.queryString));
- TermAttribute termAtt = (TermAttribute)ts.AddAttribute(typeof(TermAttribute));
+ TermAttribute termAtt = ts.AddAttribute<TermAttribute>();
int corpusNumDocs = reader.NumDocs();
Term internSavingTemplateTerm = new Term(f.fieldName); //optimization to avoid constructing new Term() objects
- Hashtable processedTerms = new Hashtable();
+ HashSet<string> processedTerms = new HashSet<string>();
while (ts.IncrementToken())
{
String term = termAtt.Term();
if (!processedTerms.Contains(term))
{
- processedTerms.Add(term,term);
+ processedTerms.Add(term);
ScoreTermQueue variantsQ = new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term
float minScore = 0;
Term startTerm = internSavingTemplateTerm.CreateTerm(term);
@@ -224,8 +225,8 @@ namespace Lucene.Net.Search
if (variantsQ.Size() < MAX_VARIANTS_PER_TERM || score > minScore)
{
ScoreTerm st = new ScoreTerm(possibleMatch, score, startTerm);
- variantsQ.Insert(st);
- minScore = ((ScoreTerm)variantsQ.Top()).score; // maintain minScore
+ variantsQ.InsertWithOverflow(st);
+ minScore = variantsQ.Top().score; // maintain minScore
}
}
}
@@ -244,9 +245,9 @@ namespace Lucene.Net.Search
int size = variantsQ.Size();
for (int i = 0; i < size; i++)
{
- ScoreTerm st = (ScoreTerm)variantsQ.Pop();
+ ScoreTerm st = variantsQ.Pop();
st.score = (st.score * st.score) * sim.Idf(df, corpusNumDocs);
- q.Insert(st);
+ q.InsertWithOverflow(st);
}
}
}
@@ -264,11 +265,6 @@ namespace Lucene.Net.Search
{
AddTerms(reader, f);
}
- //for (Iterator iter = fieldVals.iterator(); iter.hasNext(); )
- //{
- // FieldVals f = (FieldVals)iter.next();
- // addTerms(reader, f);
- //}
//clear the list of fields
fieldVals.Clear();
@@ -278,28 +274,26 @@ namespace Lucene.Net.Search
//create BooleanQueries to hold the variants for each token/field pair and ensure it
// has no coord factor
//Step 1: sort the termqueries by term/field
- Hashtable variantQueries = new Hashtable();
+ HashMap<Term, List<ScoreTerm>> variantQueries = new HashMap<Term, List<ScoreTerm>>();
int size = q.Size();
for (int i = 0; i < size; i++)
{
- ScoreTerm st = (ScoreTerm)q.Pop();
- ArrayList l = (ArrayList)variantQueries[st.fuzziedSourceTerm];
+ ScoreTerm st = q.Pop();
+ var l = variantQueries[st.fuzziedSourceTerm];
if (l == null)
{
- l = new ArrayList();
+ l = new List<ScoreTerm>();
variantQueries.Add(st.fuzziedSourceTerm, l);
}
l.Add(st);
}
//Step 2: Organize the sorted termqueries into zero-coord scoring boolean queries
- foreach(ArrayList variants in variantQueries.Values)
- //for (Iterator iter = variantQueries.values().iterator(); iter.hasNext(); )
+ foreach(var variants in variantQueries.Values)
{
- //ArrayList variants = (ArrayList)iter.next();
if (variants.Count == 1)
{
//optimize where only one selected variant
- ScoreTerm st = (ScoreTerm)variants[0];
+ ScoreTerm st = variants[0];
TermQuery tq = new FuzzyTermQuery(st.term, ignoreTF);
tq.SetBoost(st.score); // set the boost to a mix of IDF and score
bq.Add(tq, BooleanClause.Occur.SHOULD);
@@ -308,9 +302,7 @@ namespace Lucene.Net.Search
{
BooleanQuery termVariants = new BooleanQuery(true); //disable coord and IDF for these term variants
foreach(ScoreTerm st in variants)
- //for (Iterator iterator2 = variants.iterator(); iterator2.hasNext(); )
{
- //ScoreTerm st = (ScoreTerm)iterator2.next();
TermQuery tq = new FuzzyTermQuery(st.term, ignoreTF); // found a match
tq.SetBoost(st.score); // set the boost using the ScoreTerm's score
termVariants.Add(tq, BooleanClause.Occur.SHOULD); // add to query
@@ -342,7 +334,7 @@ namespace Lucene.Net.Search
}
}
- private class ScoreTermQueue : PriorityQueue
+ private class ScoreTermQueue : PriorityQueue<ScoreTerm>
{
public ScoreTermQueue(int size)
{
@@ -352,10 +344,8 @@ namespace Lucene.Net.Search
/* (non-Javadoc)
* <see cref="org.apache.lucene.util.PriorityQueue.lessThan(java.lang.Object, java.lang.Object)"/>
*/
- public override bool LessThan(Object a, Object b)
+ public override bool LessThan(ScoreTerm termA, ScoreTerm termB)
{
- ScoreTerm termA = (ScoreTerm)a;
- ScoreTerm termB = (ScoreTerm)b;
if (termA.score == termB.score)
return termA.term.CompareTo(termB.term) > 0;
else
@@ -403,28 +393,7 @@ namespace Lucene.Net.Search
//IDF is already factored into individual term boosts
return 1;
}
-
- public override float Coord(int overlap, int maxOverlap)
- {
- return base.Coord(overlap, maxOverlap);
- }
-
- public override float LengthNorm(string fieldName, int numTokens)
- {
- return base.LengthNorm(fieldName, numTokens);
- }
-
- public override float QueryNorm(float sumOfSquaredWeights)
- {
- return base.QueryNorm(sumOfSquaredWeights);
- }
-
- public override float SloppyFreq(int distance)
- {
- return base.SloppyFreq(distance);
- }
}
-
}
Modified: incubator/lucene.net/trunk/src/contrib/Queries/Similar/MoreLikeThis.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Queries/Similar/MoreLikeThis.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Queries/Similar/MoreLikeThis.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Queries/Similar/MoreLikeThis.cs Tue Feb 28 22:43:08 2012
@@ -16,8 +16,12 @@
*/
using System;
-
-using PriorityQueue = Lucene.Net.Util.PriorityQueue;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
using IndexReader = Lucene.Net.Index.IndexReader;
using Term = Lucene.Net.Index.Term;
using TermFreqVector = Lucene.Net.Index.TermFreqVector;
@@ -27,7 +31,6 @@ using TermQuery = Lucene.Net.Search.Term
using BooleanQuery = Lucene.Net.Search.BooleanQuery;
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
using Query = Lucene.Net.Search.Query;
-using Hits = Lucene.Net.Search.Hits;
using Analyzer = Lucene.Net.Analysis.Analyzer;
using TokenStream = Lucene.Net.Analysis.TokenStream;
using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
@@ -36,8 +39,6 @@ using Lucene.Net.Analysis.Tokenattribute
namespace Lucene.Net.Search.Similar
{
-
-
/// <summary> Generate "more like this" similarity queries.
/// Based on this mail:
/// <pre>
@@ -114,6 +115,8 @@ namespace Lucene.Net.Search.Similar
/// <ul>
/// <li> <see cref="SetMinTermFreq"/> </li>
/// <li> <see cref="SetMinDocFreq"/> </li>
+ /// <li> <see cref="SetMaxDocFreq"/></li>
+ /// <li> <see cref="SetMaxDocFreqPct"/></li>
/// <li> <see cref="SetMinWordLen"/> </li>
/// <li> <see cref="SetMaxWordLen"/></li>
/// <li> <see cref="SetMaxQueryTerms"/></li>
@@ -144,7 +147,7 @@ namespace Lucene.Net.Search.Similar
/// <summary> Default analyzer to parse source doc with.</summary>
/// <seealso cref="GetAnalyzer">
/// </seealso>
- public static readonly Analyzer DEFAULT_ANALYZER = new StandardAnalyzer();
+ public static readonly Analyzer DEFAULT_ANALYZER = new StandardAnalyzer(Util.Version.LUCENE_CURRENT);
/// <summary> Ignore terms with less than this frequency in the source doc.</summary>
/// <seealso cref="GetMinTermFreq">
@@ -158,7 +161,15 @@ namespace Lucene.Net.Search.Similar
/// </seealso>
/// <seealso cref="SetMinDocFreq">
/// </seealso>
- public const int DEFALT_MIN_DOC_FREQ = 5;
+ public const int DEFAULT_MIN_DOC_FREQ = 5;
+
+ /// <summary>
+ /// Ignore words wich occur in more than this many docs
+ /// </summary>
+ /// <seealso cref="GetMaxDocFreq"/>
+ /// <seealso cref="SetMaxDocFreq"/>
+ /// <seealso cref="SetMaxDocFreqPct"/>
+ public const int DEFAULT_MAX_DOC_FREQ = int.MaxValue;
/// <summary> Boost terms in query based on score.</summary>
/// <seealso cref="IsBoost">
@@ -194,10 +205,10 @@ namespace Lucene.Net.Search.Similar
/// </seealso>
/// <seealso cref="GetStopWords">
/// </seealso>
- public static readonly System.Collections.Hashtable DEFAULT_STOP_WORDS = null;
+ public static readonly ISet<string> DEFAULT_STOP_WORDS = null;
/// <summary> Current set of stop words.</summary>
- private System.Collections.Hashtable stopWords = DEFAULT_STOP_WORDS;
+ private ISet<string> stopWords = DEFAULT_STOP_WORDS;
/// <summary> Return a Query with no more than this many terms.
///
@@ -217,7 +228,12 @@ namespace Lucene.Net.Search.Similar
private int minTermFreq = DEFAULT_MIN_TERM_FREQ;
/// <summary> Ignore words which do not occur in at least this many docs.</summary>
- private int minDocFreq = DEFALT_MIN_DOC_FREQ;
+ private int minDocFreq = DEFAULT_MIN_DOC_FREQ;
+
+ /// <summary>
+ /// Ignore words which occur in more than this many docs.
+ /// </summary>
+ private int maxDocfreq = DEFAULT_MAX_DOC_FREQ;
/// <summary> Should we apply a boost to the Query based on the scores?</summary>
private bool boost = DEFAULT_BOOST;
@@ -228,8 +244,6 @@ namespace Lucene.Net.Search.Similar
/// <summary> The maximum number of tokens to parse in each example doc field that is not stored with TermVector support</summary>
private int maxNumTokensParsed = DEFAULT_MAX_NUM_TOKENS_PARSED;
-
-
/// <summary> Ignore words if less than this len.</summary>
private int minWordLen = DEFAULT_MIN_WORD_LENGTH;
@@ -266,7 +280,7 @@ namespace Lucene.Net.Search.Similar
}
/// <summary> Constructor requiring an IndexReader.</summary>
- public MoreLikeThis(IndexReader ir) : this(ir,new DefaultSimilarity() )
+ public MoreLikeThis(IndexReader ir) : this(ir,new DefaultSimilarity())
{
}
@@ -332,7 +346,7 @@ namespace Lucene.Net.Search.Similar
}
/// <summary> Returns the frequency at which words will be ignored which do not occur in at least this
- /// many docs. The default frequency is <see cref="DEFALT_MIN_DOC_FREQ"/>.
+ /// many docs. The default frequency is <see cref="DEFAULT_MIN_DOC_FREQ"/>.
///
/// </summary>
/// <returns> the frequency at which words will be ignored which do not occur in at least this
@@ -355,6 +369,43 @@ namespace Lucene.Net.Search.Similar
this.minDocFreq = minDocFreq;
}
+ /// <summary>
+ /// Returns the maximum frequency in which words may still appear.
+ /// Words that appear in more than this many docs will be ignored. The default frequency is
+ /// <see cref="DEFAULT_MAX_DOC_FREQ"/>
+ /// </summary>
+ /// <returns>get the maximum frequency at which words are still allowed,
+ /// words which occur in more docs than this are ignored.</returns>
+ public int GetMaxDocFreq()
+ {
+ return this.maxDocfreq;
+ }
+
+ /// <summary>
+ /// Set the maximum frequency in which words may still appear. Words that appear
+ /// in more than this many docs will be ignored.
+ /// </summary>
+ /// <param name="maxFreq">
+ /// the maximum count of documents that a term may appear
+ /// in to be still considered relevant</param>
+ public void SetMaxDocFreq(int maxFreq)
+ {
+ this.maxDocfreq = maxFreq;
+ }
+
+ /// <summary>
+ /// Set the maximum percentage in which words may still appear. Words that appear
+ /// in more than this many percent of all docs will be ignored.
+ /// </summary>
+ /// <param name="maxPercentage">
+ /// the maximum percentage of documents (0-100) that a term may appear
+ /// in to be still considered relevant
+ /// </param>
+ public void SetMaxDocFreqPct(int maxPercentage)
+ {
+ this.maxDocfreq = maxPercentage * ir.NumDocs() / 100;
+ }
+
/// <summary> Returns whether to boost terms in query based on "score" or not. The default is
/// <see cref="DEFAULT_BOOST"/>.
///
@@ -459,7 +510,7 @@ namespace Lucene.Net.Search.Similar
/// </seealso>
/// <seealso cref="GetStopWords">
/// </seealso>
- public void SetStopWords(System.Collections.Hashtable stopWords)
+ public void SetStopWords(ISet<string> stopWords)
{
this.stopWords = stopWords;
}
@@ -467,7 +518,7 @@ namespace Lucene.Net.Search.Similar
/// <summary> Get the current stop words being used.</summary>
/// <seealso cref="SetStopWords">
/// </seealso>
- public System.Collections.Hashtable GetStopWords()
+ public ISet<string> GetStopWords()
{
return stopWords;
}
@@ -511,24 +562,16 @@ namespace Lucene.Net.Search.Similar
maxNumTokensParsed = i;
}
-
-
-
- /// <summary> Return a query that will return docs like the passed lucene document ID.
- ///
- /// </summary>
- /// <param name="docNum">the documentID of the lucene doc to generate the 'More Like This" query for.
- /// </param>
- /// <returns> a query that will return docs like the passed lucene document ID.
- /// </returns>
+ /// <summary>Return a query that will return docs like the passed lucene document ID.</summary>
+ /// <param name="docNum">the documentID of the lucene doc to generate the 'More Like This" query for.</param>
+ /// <returns> a query that will return docs like the passed lucene document ID.</returns>
public Query Like(int docNum)
{
if (fieldNames == null)
{
// gather list of valid fields from lucene
- System.Collections.Generic.ICollection<string> fields = ir.GetFieldNames(IndexReader.FieldOption.INDEXED);
- fieldNames = new string[fields.Count];
- fields.CopyTo(fieldNames, 0);
+ ICollection<string> fields = ir.GetFieldNames(IndexReader.FieldOption.INDEXED);
+ fieldNames = fields.ToArray();
}
return CreateQuery(RetrieveTerms(docNum));
@@ -544,9 +587,8 @@ namespace Lucene.Net.Search.Similar
if (fieldNames == null)
{
// gather list of valid fields from lucene
- System.Collections.Generic.ICollection<string> fields = ir.GetFieldNames(IndexReader.FieldOption.INDEXED);
- fieldNames = new string[fields.Count];
- fields.CopyTo(fieldNames, 0);
+ ICollection<string> fields = ir.GetFieldNames(IndexReader.FieldOption.INDEXED);
+ fieldNames = fields.ToArray();
}
return Like(new System.IO.StreamReader(f.FullName, System.Text.Encoding.Default));
@@ -559,7 +601,7 @@ namespace Lucene.Net.Search.Similar
/// </returns>
public Query Like(System.Uri u)
{
- return Like(new System.IO.StreamReader(((System.Net.HttpWebRequest)System.Net.WebRequest.Create(u)).GetResponse().GetResponseStream(), System.Text.Encoding.Default));
+ return Like(new System.IO.StreamReader((System.Net.WebRequest.Create(u)).GetResponse().GetResponseStream(), System.Text.Encoding.Default));
}
/// <summary> Return a query that will return docs like the passed stream.
@@ -583,7 +625,7 @@ namespace Lucene.Net.Search.Similar
}
/// <summary> Create the More like query from a PriorityQueue</summary>
- private Query CreateQuery(PriorityQueue q)
+ private Query CreateQuery(PriorityQueue<object[]> q)
{
BooleanQuery query = new BooleanQuery();
System.Object cur;
@@ -599,9 +641,9 @@ namespace Lucene.Net.Search.Similar
{
if (qterms == 0)
{
- bestScore = (float)((System.Single)ar[2]);
+ bestScore = (float)ar[2];
}
- float myScore = (float)((System.Single)ar[2]);
+ float myScore = (float)ar[2];
tq.SetBoost(boostFactor * myScore / bestScore);
}
@@ -630,19 +672,19 @@ namespace Lucene.Net.Search.Similar
/// </summary>
/// <param name="words">a map of words keyed on the word(String) with Int objects as the values.
/// </param>
- private PriorityQueue CreateQueue(System.Collections.IDictionary words)
+ private PriorityQueue<object[]> CreateQueue(IDictionary<string,Int> words)
{
// have collected all words in doc and their freqs
int numDocs = ir.NumDocs();
FreqQ res = new FreqQ(words.Count); // will order words by score
- System.Collections.IEnumerator it = words.Keys.GetEnumerator();
+ var it = words.Keys.GetEnumerator();
while (it.MoveNext())
{
// for every word
- System.String word = (System.String)it.Current;
+ System.String word = it.Current;
- int tf = ((Int)words[word]).x; // term freq in the source doc
+ int tf = words[word].x; // term freq in the source doc
if (minTermFreq > 0 && tf < minTermFreq)
{
continue; // filter out words that don't occur enough times in the source
@@ -663,6 +705,11 @@ namespace Lucene.Net.Search.Similar
continue; // filter out words that don't occur in enough docs
}
+ if (docFreq > maxDocfreq)
+ {
+ continue; // filter out words that occur in too many docs
+ }
+
if (docFreq == 0)
{
continue; // index update problem?
@@ -672,7 +719,7 @@ namespace Lucene.Net.Search.Similar
float score = tf * idf;
// only really need 1st 3 entries, other ones are for troubleshooting
- res.Insert(new System.Object[] { word, topField, (float)score, (float)idf, (System.Int32)docFreq, (System.Int32)tf });
+ res.InsertWithOverflow(new System.Object[] { word, topField, score, idf, docFreq, tf });
}
return res;
}
@@ -728,7 +775,8 @@ namespace Lucene.Net.Search.Similar
temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
temp_writer.AutoFlush = true;
System.IO.StreamWriter o = temp_writer;
- IndexReader r = IndexReader.Open(indexName);
+ FSDirectory dir = FSDirectory.Open(new DirectoryInfo(indexName));
+ IndexReader r = IndexReader.Open(dir, true);
o.WriteLine("Open index " + indexName + " which has " + r.NumDocs() + " docs");
MoreLikeThis mlt = new MoreLikeThis(r);
@@ -751,17 +799,18 @@ namespace Lucene.Net.Search.Similar
o.WriteLine("q: " + query);
o.WriteLine();
- IndexSearcher searcher = new IndexSearcher(indexName);
+ IndexSearcher searcher = new IndexSearcher(dir, true);
- Hits hits = searcher.Search(query);
- int len = hits.Length();
+ TopDocs hits = searcher.Search(query, null, 25);
+ int len = hits.TotalHits;
o.WriteLine("found: " + len + " documents matching");
o.WriteLine();
+ ScoreDoc[] scoreDocs = hits.ScoreDocs;
for (int i = 0; i < System.Math.Min(25, len); i++)
{
- Document d = hits.Doc(i);
+ Document d = searcher.Doc(scoreDocs[i].doc);
System.String summary = d.Get("summary");
- o.WriteLine("score : " + hits.Score(i));
+ o.WriteLine("score : " + scoreDocs[i].score);
o.WriteLine("url : " + d.Get("url"));
o.WriteLine("\ttitle : " + d.Get("title"));
if (summary != null)
@@ -775,9 +824,9 @@ namespace Lucene.Net.Search.Similar
/// </summary>
/// <param name="docNum">the id of the lucene document from which to find terms
/// </param>
- private PriorityQueue RetrieveTerms(int docNum)
+ private PriorityQueue<object[]> RetrieveTerms(int docNum)
{
- System.Collections.IDictionary termFreqMap = new System.Collections.Hashtable();
+ IDictionary<string,Int> termFreqMap = new HashMap<string,Int>();
for (int i = 0; i < fieldNames.Length; i++)
{
System.String fieldName = fieldNames[i];
@@ -810,7 +859,7 @@ namespace Lucene.Net.Search.Similar
/// </param>
/// <param name="vector">List of terms and their frequencies for a doc/field
/// </param>
- private void AddTermFrequencies(System.Collections.IDictionary termFreqMap, TermFreqVector vector)
+ private void AddTermFrequencies(IDictionary<string, Int> termFreqMap, TermFreqVector vector)
{
System.String[] terms = vector.GetTerms();
int[] freqs = vector.GetTermFrequencies();
@@ -823,7 +872,7 @@ namespace Lucene.Net.Search.Similar
continue;
}
// increment frequency
- Int cnt = (Int)termFreqMap[term];
+ Int cnt = termFreqMap[term];
if (cnt == null)
{
cnt = new Int();
@@ -843,12 +892,12 @@ namespace Lucene.Net.Search.Similar
/// </param>
/// <param name="fieldName">Used by analyzer for any special per-field analysis
/// </param>
- private void AddTermFrequencies(System.IO.TextReader r, System.Collections.IDictionary termFreqMap, System.String fieldName)
+ private void AddTermFrequencies(System.IO.TextReader r, IDictionary<string,Int> termFreqMap, System.String fieldName)
{
TokenStream ts = analyzer.TokenStream(fieldName, r);
int tokenCount=0;
// for every token
- TermAttribute termAtt = (TermAttribute) ts.AddAttribute(typeof(TermAttribute));
+ TermAttribute termAtt = ts.AddAttribute<TermAttribute>();
while (ts.IncrementToken()) {
string word = termAtt.Term();
@@ -862,7 +911,7 @@ namespace Lucene.Net.Search.Similar
}
// increment frequency
- Int cnt = (Int) termFreqMap[word];
+ Int cnt = termFreqMap[word];
if (cnt == null) {
termFreqMap[word] = new Int();
}
@@ -923,9 +972,9 @@ namespace Lucene.Net.Search.Similar
/// </returns>
/// <seealso cref="RetrieveInterestingTerms(System.IO.TextReader)">
/// </seealso>
- public PriorityQueue RetrieveTerms(System.IO.TextReader r)
+ public PriorityQueue<object[]> RetrieveTerms(System.IO.TextReader r)
{
- System.Collections.IDictionary words = new System.Collections.Hashtable();
+ IDictionary<string, Int> words = new HashMap<string,Int>();
for (int i = 0; i < fieldNames.Length; i++)
{
System.String fieldName = fieldNames[i];
@@ -937,8 +986,8 @@ namespace Lucene.Net.Search.Similar
public System.String[] RetrieveInterestingTerms(int docNum)
{
- System.Collections.ArrayList al = new System.Collections.ArrayList(maxQueryTerms);
- PriorityQueue pq = RetrieveTerms(docNum);
+ List<object> al = new List<object>(maxQueryTerms);
+ PriorityQueue<object[]> pq = RetrieveTerms(docNum);
System.Object cur;
int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all words but that's probably not useful to our caller...
// we just want to return the top words
@@ -947,9 +996,9 @@ namespace Lucene.Net.Search.Similar
System.Object[] ar = (System.Object[])cur;
al.Add(ar[0]); // the 1st entry is the interesting word
}
- System.String[] res = new System.String[al.Count];
- // return (System.String[]) SupportClass.ICollectionSupport.ToArray(al, res);
- return (System.String[])al.ToArray(typeof(System.String));
+ //System.String[] res = new System.String[al.Count];
+ //return al.toArray(res);
+ return al.Select(x => x.ToString()).ToArray();
}
/// <summary> Convenience routine to make it easy to return the most interesting words in a document.
@@ -966,8 +1015,8 @@ namespace Lucene.Net.Search.Similar
/// </seealso>
public System.String[] RetrieveInterestingTerms(System.IO.TextReader r)
{
- System.Collections.ArrayList al = new System.Collections.ArrayList(maxQueryTerms);
- PriorityQueue pq = RetrieveTerms(r);
+ List<object> al = new List<object>(maxQueryTerms);
+ PriorityQueue<object[]> pq = RetrieveTerms(r);
System.Object cur;
int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all words but that's probably not useful to our caller...
// we just want to return the top words
@@ -976,25 +1025,23 @@ namespace Lucene.Net.Search.Similar
System.Object[] ar = (System.Object[])cur;
al.Add(ar[0]); // the 1st entry is the interesting word
}
- System.String[] res = new System.String[al.Count];
+ //System.String[] res = new System.String[al.Count];
// return (System.String[]) SupportClass.ICollectionSupport.ToArray(al, res);
- return (System.String[])al.ToArray(typeof(System.String));
+ return al.Select(x => x.ToString()).ToArray();
}
/// <summary> PriorityQueue that orders words by score.</summary>
- private class FreqQ : PriorityQueue
+ private class FreqQ : PriorityQueue<object[]>
{
internal FreqQ(int s)
{
Initialize(s);
}
- override public bool LessThan(System.Object a, System.Object b)
+ override public bool LessThan(System.Object[] aa, System.Object[] bb)
{
- System.Object[] aa = (System.Object[])a;
- System.Object[] bb = (System.Object[])b;
- System.Single fa = (System.Single)aa[2];
- System.Single fb = (System.Single)bb[2];
+ float fa = (float)aa[2];
+ float fb = (float)bb[2];
return (float)fa > (float)fb;
}
}
Modified: incubator/lucene.net/trunk/src/contrib/Queries/Similar/MoreLikeThisQuery.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Queries/Similar/MoreLikeThisQuery.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Queries/Similar/MoreLikeThisQuery.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Queries/Similar/MoreLikeThisQuery.cs Tue Feb 28 22:43:08 2012
@@ -35,15 +35,13 @@ namespace Lucene.Net.Search.Similar
*/
public class MoreLikeThisQuery : Query
{
-
-
private String likeText;
private String[] moreLikeFields;
private Analyzer analyzer;
float percentTermsToMatch = 0.3f;
int minTermFrequency = 1;
int maxQueryTerms = 5;
- System.Collections.Hashtable stopWords = null;
+ ISet<string> stopWords = null;
int minDocFreq = -1;
@@ -144,11 +142,11 @@ namespace Lucene.Net.Search.Similar
{
this.moreLikeFields = moreLikeFields;
}
- public System.Collections.Hashtable GetStopWords()
+ public ISet<string> GetStopWords()
{
return stopWords;
}
- public void SetStopWords(System.Collections.Hashtable stopWords)
+ public void SetStopWords(ISet<string> stopWords)
{
this.stopWords = stopWords;
}
Modified: incubator/lucene.net/trunk/src/contrib/Queries/Similar/SimilarityQueries.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Queries/Similar/SimilarityQueries.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Queries/Similar/SimilarityQueries.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Queries/Similar/SimilarityQueries.cs Tue Feb 28 22:43:08 2012
@@ -16,6 +16,7 @@
*/
using System;
+using System.Collections.Generic;
using Analyzer = Lucene.Net.Analysis.Analyzer;
using TokenStream = Lucene.Net.Analysis.TokenStream;
using Term = Lucene.Net.Index.Term;
@@ -82,13 +83,13 @@ namespace Similarity.Net
/// <returns> a query with all unique words in 'body'
/// </returns>
/// <throws> IOException this can't happen... </throws>
- public static Query FormSimilarQuery(System.String body, Analyzer a, System.String field, System.Collections.Hashtable stop)
+ public static Query FormSimilarQuery(System.String body, Analyzer a, System.String field, ISet<string> stop)
{
TokenStream ts = a.TokenStream(field, new System.IO.StringReader(body));
- TermAttribute termAtt = (TermAttribute)ts.AddAttribute(typeof(TermAttribute));
+ TermAttribute termAtt = ts.AddAttribute<TermAttribute>();
BooleanQuery tmp = new BooleanQuery();
- System.Collections.Hashtable already = new System.Collections.Hashtable(); // ignore dups
+ ISet<string> already = new HashSet<string>(); // ignore dups
while (ts.IncrementToken())
{
String word = termAtt.Term();
@@ -96,9 +97,9 @@ namespace Similarity.Net
if (stop != null && stop.Contains(word))
continue;
// ignore dups
- if (already.Contains(word) == true)
+ if (already.Contains(word))
continue;
- already.Add(word, word);
+ already.Add(word);
// add to query
TermQuery tq = new TermQuery(new Term(field, word));
try
Modified: incubator/lucene.net/trunk/src/contrib/Queries/TermsFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Queries/TermsFilter.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Queries/TermsFilter.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Queries/TermsFilter.cs Tue Feb 28 22:43:08 2012
@@ -33,7 +33,7 @@ namespace Lucene.Net.Search
/// <summary>
/// The set of terms for this filter.
/// </summary>
- protected HashSet<Term> terms = new HashSet<Term>();
+ protected ISet<Term> terms = new SortedSet<Term>();
/// <summary>
/// Add a term to the set.
@@ -83,6 +83,7 @@ namespace Lucene.Net.Search
return false;
}
TermsFilter test = (TermsFilter)obj;
+ // TODO: Does SortedSet have an issues like List<T>? see EquatableList in Support
return (terms == test.terms || (terms != null && terms.Equals(test.terms)));
}
Modified: incubator/lucene.net/trunk/src/contrib/Regex/RegexQuery.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Regex/RegexQuery.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Regex/RegexQuery.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Regex/RegexQuery.cs Tue Feb 28 22:43:08 2012
@@ -16,7 +16,9 @@
*/
using System;
+using System.Text;
using Lucene.Net.Index;
+using Lucene.Net.Util;
namespace Lucene.Net.Search.Regex
{
@@ -27,16 +29,17 @@ namespace Lucene.Net.Search.Regex
public class RegexQuery : MultiTermQuery, IRegexQueryCapable, IEquatable<RegexQuery>
{
private IRegexCapabilities _regexImpl = new CSharpRegexCapabilities();
+ public Term Term { get; private set; }
- public RegexQuery(Term term) : base(term)
+ public RegexQuery(Term term)
{
+ Term = term;
}
/// <summary>Construct the enumeration to be used, expanding the pattern term. </summary>
public override FilteredTermEnum GetEnum(IndexReader reader)
{
- Term term = new Term(GetTerm().Field(), GetTerm().Text());
- return new RegexTermEnum(reader, term, _regexImpl);
+ return new RegexTermEnum(reader, Term, _regexImpl);
}
public void SetRegexImplementation(IRegexCapabilities impl)
@@ -49,7 +52,21 @@ namespace Lucene.Net.Search.Regex
return _regexImpl;
}
- /// <summary>
+
+ public override String ToString(String field)
+ {
+ StringBuilder buffer = new StringBuilder();
+ if (!Term.Field().Equals(field))
+ {
+ buffer.Append(Term.Field());
+ buffer.Append(":");
+ }
+ buffer.Append(Term.Text());
+ buffer.Append(ToStringUtils.Boost(GetBoost()));
+ return buffer.ToString();
+ }
+
+ /// <summary>
/// Indicates whether the current object is equal to another object of the same type.
/// </summary>
/// <returns>
Modified: incubator/lucene.net/trunk/src/contrib/Regex/SpanRegexQuery.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Regex/SpanRegexQuery.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Regex/SpanRegexQuery.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Regex/SpanRegexQuery.cs Tue Feb 28 22:43:08 2012
@@ -97,15 +97,9 @@ namespace Lucene.Net.Search.Regex
return _term.Field();
}
- /// <summary>Returns a collection of all terms matched by this query.</summary>
- /// <deprecated> use extractTerms instead
- /// </deprecated>
- /// <seealso cref="Query.ExtractTerms">
- /// </seealso>
- [Obsolete("use ExtractTerms instead")]
- public override ICollection GetTerms()
+ public ICollection<Term> GetTerms()
{
- ArrayList terms = new ArrayList {_term};
+ ICollection<Term> terms = new List<Term>(){_term};
return terms;
}
Modified: incubator/lucene.net/trunk/src/contrib/Similarity/Similar/MoreLikeThis.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Similarity/Similar/MoreLikeThis.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Similarity/Similar/MoreLikeThis.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Similarity/Similar/MoreLikeThis.cs Tue Feb 28 22:43:08 2012
@@ -16,8 +16,11 @@
*/
using System;
-
-using PriorityQueue = Lucene.Net.Util.PriorityQueue;
+using System.IO;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
+using Lucene.Net.Util;
using IndexReader = Lucene.Net.Index.IndexReader;
using Term = Lucene.Net.Index.Term;
using TermFreqVector = Lucene.Net.Index.TermFreqVector;
@@ -27,11 +30,11 @@ using TermQuery = Lucene.Net.Search.Term
using BooleanQuery = Lucene.Net.Search.BooleanQuery;
using IndexSearcher = Lucene.Net.Search.IndexSearcher;
using Query = Lucene.Net.Search.Query;
-using Hits = Lucene.Net.Search.Hits;
using Analyzer = Lucene.Net.Analysis.Analyzer;
using TokenStream = Lucene.Net.Analysis.TokenStream;
using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
using Document = Lucene.Net.Documents.Document;
+using Version = Lucene.Net.Util.Version;
namespace Similarity.Net
{
@@ -150,7 +153,7 @@ namespace Similarity.Net
/// <summary> Default analyzer to parse source doc with.</summary>
/// <seealso cref="GetAnalyzer">
/// </seealso>
- public static readonly Analyzer DEFAULT_ANALYZER = new StandardAnalyzer();
+ public static readonly Analyzer DEFAULT_ANALYZER = new StandardAnalyzer(Version.LUCENE_CURRENT);
/// <summary> Ignore terms with less than this frequency in the source doc.</summary>
/// <seealso cref="GetMinTermFreq">
@@ -558,7 +561,7 @@ namespace Similarity.Net
}
/// <summary> Create the More like query from a PriorityQueue</summary>
- private Query CreateQuery(PriorityQueue q)
+ private Query CreateQuery(PriorityQueue<object[]> q)
{
BooleanQuery query = new BooleanQuery();
System.Object cur;
@@ -605,7 +608,7 @@ namespace Similarity.Net
/// </summary>
/// <param name="words">a map of words keyed on the word(String) with Int objects as the values.
/// </param>
- private PriorityQueue CreateQueue(System.Collections.IDictionary words)
+ private PriorityQueue<object[]> CreateQueue(System.Collections.IDictionary words)
{
// have collected all words in doc and their freqs
int numDocs = ir.NumDocs();
@@ -647,7 +650,7 @@ namespace Similarity.Net
float score = tf * idf;
// only really need 1st 3 entries, other ones are for troubleshooting
- res.Insert(new System.Object[]{word, topField, (float) score, (float) idf, (System.Int32) docFreq, (System.Int32) tf});
+ res.InsertWithOverflow(new System.Object[]{word, topField, score, idf, docFreq, tf});
}
return res;
}
@@ -699,11 +702,10 @@ namespace Similarity.Net
}
}
- System.IO.StreamWriter temp_writer;
- temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
- temp_writer.AutoFlush = true;
- System.IO.StreamWriter o = temp_writer;
- IndexReader r = IndexReader.Open(indexName);
+ var o = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding)
+ {AutoFlush = true};
+ var dir = FSDirectory.Open(new DirectoryInfo(indexName));
+ IndexReader r = IndexReader.Open(dir, true);
o.WriteLine("Open index " + indexName + " which has " + r.NumDocs() + " docs");
MoreLikeThis mlt = new MoreLikeThis(r);
@@ -726,17 +728,19 @@ namespace Similarity.Net
o.WriteLine("q: " + query);
o.WriteLine();
- IndexSearcher searcher = new IndexSearcher(indexName);
+ IndexSearcher searcher = new IndexSearcher(dir, true);
- Hits hits = searcher.Search(query);
- int len = hits.Length();
+ TopDocs hits = searcher.Search(query, null, 25);
+ int len = hits.TotalHits;
o.WriteLine("found: " + len + " documents matching");
o.WriteLine();
+
+ ScoreDoc[] scoreDocs = hits.ScoreDocs;
for (int i = 0; i < System.Math.Min(25, len); i++)
{
- Document d = hits.Doc(i);
+ Document d = searcher.Doc(scoreDocs[i].doc);
System.String summary = d.Get("summary");
- o.WriteLine("score : " + hits.Score(i));
+ o.WriteLine("score : " + scoreDocs[i].score);
o.WriteLine("url : " + d.Get("url"));
o.WriteLine("\ttitle : " + d.Get("title"));
if (summary != null)
@@ -750,7 +754,7 @@ namespace Similarity.Net
/// </summary>
/// <param name="docNum">the id of the lucene document from which to find terms
/// </param>
- private PriorityQueue RetrieveTerms(int docNum)
+ private PriorityQueue<object[]> RetrieveTerms(int docNum)
{
System.Collections.IDictionary termFreqMap = new System.Collections.Hashtable();
for (int i = 0; i < fieldNames.Length; i++)
@@ -821,33 +825,30 @@ namespace Similarity.Net
private void AddTermFrequencies(System.IO.TextReader r, System.Collections.IDictionary termFreqMap, System.String fieldName)
{
TokenStream ts = analyzer.TokenStream(fieldName, r);
- Lucene.Net.Analysis.Token token;
- int tokenCount = 0;
- while ((token = ts.Next()) != null)
- {
- // for every token
- System.String word = token.TermText();
- tokenCount++;
- if (tokenCount > maxNumTokensParsed)
- {
- break;
- }
- if (IsNoiseWord(word))
- {
- continue;
- }
-
- // increment frequency
- Int cnt = (Int) termFreqMap[word];
- if (cnt == null)
- {
- termFreqMap[word] = new Int();
- }
- else
- {
- cnt.x++;
- }
- }
+ int tokenCount=0;
+ // for every token
+ var termAtt = ts.AddAttribute<TermAttribute>();
+
+ while (ts.IncrementToken()) {
+ String word = termAtt.Term();
+ tokenCount++;
+ if(tokenCount>maxNumTokensParsed)
+ {
+ break;
+ }
+ if(IsNoiseWord(word)){
+ continue;
+ }
+
+ // increment frequency
+ Int cnt = (Int)termFreqMap[word];
+ if (cnt == null) {
+ termFreqMap[word] = new Int();
+ }
+ else {
+ cnt.x++;
+ }
+ }
}
@@ -901,7 +902,7 @@ namespace Similarity.Net
/// </returns>
/// <seealso cref="RetrieveInterestingTerms">
/// </seealso>
- public PriorityQueue RetrieveTerms(System.IO.StreamReader r)
+ public PriorityQueue<object[]> RetrieveTerms(System.IO.StreamReader r)
{
System.Collections.IDictionary words = new System.Collections.Hashtable();
for (int i = 0; i < fieldNames.Length; i++)
@@ -927,7 +928,7 @@ namespace Similarity.Net
public System.String[] RetrieveInterestingTerms(System.IO.StreamReader r)
{
System.Collections.ArrayList al = new System.Collections.ArrayList(maxQueryTerms);
- PriorityQueue pq = RetrieveTerms(r);
+ PriorityQueue<object[]> pq = RetrieveTerms(r);
System.Object cur;
int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all words but that's probably not useful to our caller...
// we just want to return the top words
@@ -942,19 +943,17 @@ namespace Similarity.Net
}
/// <summary> PriorityQueue that orders words by score.</summary>
- private class FreqQ : PriorityQueue
+ private class FreqQ : PriorityQueue<object[]>
{
internal FreqQ(int s)
{
Initialize(s);
}
- override public bool LessThan(System.Object a, System.Object b)
+ override public bool LessThan(System.Object[] a, System.Object[] b)
{
- System.Object[] aa = (System.Object[]) a;
- System.Object[] bb = (System.Object[]) b;
- System.Single fa = (System.Single) aa[2];
- System.Single fb = (System.Single) bb[2];
+ System.Single fa = (System.Single) a[2];
+ System.Single fb = (System.Single) b[2];
return (float) fa > (float) fb;
}
}
Modified: incubator/lucene.net/trunk/src/contrib/Similarity/Similar/SimilarityQueries.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Similarity/Similar/SimilarityQueries.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Similarity/Similar/SimilarityQueries.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Similarity/Similar/SimilarityQueries.cs Tue Feb 28 22:43:08 2012
@@ -16,7 +16,9 @@
*/
using System;
-
+using System.Collections.Generic;
+using System.IO;
+using Lucene.Net.Analysis.Tokenattributes;
using Analyzer = Lucene.Net.Analysis.Analyzer;
using TokenStream = Lucene.Net.Analysis.TokenStream;
using Term = Lucene.Net.Index.Term;
@@ -28,7 +30,7 @@ using BooleanClause = Lucene.Net.Search.
namespace Similarity.Net
{
-
+
/// <summary> Simple similarity measures.
///
///
@@ -41,7 +43,7 @@ namespace Similarity.Net
private SimilarityQueries()
{
}
-
+
/// <summary> Simple similarity query generators.
/// Takes every unique word and forms a boolean query where all words are optional.
/// After you get this you'll use to to query your <see cref="IndexSearcher"/> for similar docs.
@@ -86,27 +88,27 @@ namespace Similarity.Net
/// <returns> a query with all unique words in 'body'
/// </returns>
/// <throws> IOException this can't happen... </throws>
- public static Query FormSimilarQuery(System.String body, Analyzer a, System.String field, System.Collections.Hashtable stop)
+ public static Query FormSimilarQuery(System.String body, Analyzer a, System.String field,
+ System.Collections.Hashtable stop)
{
- TokenStream ts = a.TokenStream(field, new System.IO.StringReader(body));
- Lucene.Net.Analysis.Token t;
- BooleanQuery tmp = new BooleanQuery();
- System.Collections.Hashtable already = new System.Collections.Hashtable(); // ignore dups
- while ((t = ts.Next()) != null)
+ TokenStream ts = a.TokenStream(field, new StringReader(body));
+ var termAtt = ts.AddAttribute<TermAttribute>();
+
+ var tmp = new BooleanQuery();
+ var already = new HashSet<String>(); // ignore dups
+ while (ts.IncrementToken())
{
- System.String word = t.TermText();
+ String word = termAtt.Term();
// ignore opt stop words
- if (stop != null && stop.Contains(word))
- continue;
+ if (stop != null &&
+ stop.Contains(word)) continue;
// ignore dups
- if (already.Contains(word) == true)
- continue;
- already.Add(word, word);
+ if (!already.Add(word)) continue;
// add to query
- TermQuery tq = new TermQuery(new Term(field, word));
+ var tq = new TermQuery(new Term(field, word));
try
{
- tmp.Add(tq, BooleanClause.Occur.SHOULD); //false, false);
+ tmp.Add(tq, BooleanClause.Occur.SHOULD);
}
catch (BooleanQuery.TooManyClauses too)
{
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballAnalyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballAnalyzer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballAnalyzer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballAnalyzer.cs Tue Feb 28 22:43:08 2012
@@ -16,48 +16,110 @@
*/
using System;
+using System.Collections.Generic;
+using System.IO;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using SF.Snowball.Ext;
+using Version = Lucene.Net.Util.Version;
+
namespace Lucene.Net.Analysis.Snowball
{
-
- /// <summary>Filters <see cref="StandardTokenizer"/> with <see cref="StandardFilter"/>, {@link
- /// LowerCaseFilter}, <see cref="StopFilter"/> and <see cref="SnowballFilter"/>.
- ///
- /// Available stemmers are listed in <see cref="SF.Snowball.Ext"/>. The name of a
- /// stemmer is the part of the class name before "Stemmer", e.g., the stemmer in
- /// <see cref="EnglishStemmer"/> is named "English".
- /// </summary>
- public class SnowballAnalyzer : Analyzer
- {
- private System.String name;
- private System.Collections.Hashtable stopSet;
-
- /// <summary>Builds the named analyzer with no stop words. </summary>
- public SnowballAnalyzer(System.String name)
- {
- this.name = name;
- }
-
- /// <summary>Builds the named analyzer with the given stop words. </summary>
- public SnowballAnalyzer(System.String name, System.String[] stopWords) : this(name)
- {
- stopSet = StopFilter.MakeStopSet(stopWords);
- }
-
- /// <summary>Constructs a <see cref="StandardTokenizer"/> filtered by a {@link
- /// StandardFilter}, a <see cref="LowerCaseFilter"/> and a <see cref="StopFilter"/>.
- /// </summary>
+
+ /// <summary>Filters <see cref="StandardTokenizer"/> with <see cref="StandardFilter"/>, {@link
+ /// LowerCaseFilter}, <see cref="StopFilter"/> and <see cref="SnowballFilter"/>.
+ ///
+ /// Available stemmers are listed in <see cref="SF.Snowball.Ext"/>. The name of a
+ /// stemmer is the part of the class name before "Stemmer", e.g., the stemmer in
+ /// <see cref="EnglishStemmer"/> is named "English".
+ ///
+ /// <p><b>NOTE:</b> This class uses the same <see cref="Version"/>
+ /// dependent settings as <see cref="StandardAnalyzer"/></p>
+ /// </summary>
+ public class SnowballAnalyzer : Analyzer
+ {
+ private System.String name;
+ private ISet<string> stopSet;
+ private readonly Version matchVersion;
+
+ /// <summary>Builds the named analyzer with no stop words. </summary>
+ public SnowballAnalyzer(Version matchVersion, System.String name)
+ {
+ this.name = name;
+ SetOverridesTokenStreamMethod<SnowballAnalyzer>();
+ this.matchVersion = matchVersion;
+ }
+
+ /// <summary>Builds the named analyzer with the given stop words. </summary>
+ [Obsolete("Use SnowballAnalyzer(Version, string, ISet) instead.")]
+ public SnowballAnalyzer(Version matchVersion, System.String name, System.String[] stopWords)
+ : this(matchVersion, name)
+ {
+ stopSet = StopFilter.MakeStopSet(stopWords);
+ }
+
+ /// <summary>
+ /// Builds the named analyzer with the given stop words.
+ /// </summary>
+ public SnowballAnalyzer(Version matchVersion, string name, ISet<string> stopWords)
+ : this(matchVersion, name)
+ {
+ stopSet = CharArraySet.UnmodifiableSet(CharArraySet.Copy(stopWords));
+ }
+
+ /// <summary>Constructs a <see cref="StandardTokenizer"/> filtered by a {@link
+ /// StandardFilter}, a <see cref="LowerCaseFilter"/> and a <see cref="StopFilter"/>.
+ /// </summary>
public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
- {
- TokenStream result = new StandardTokenizer(reader);
- result = new StandardFilter(result);
- result = new LowerCaseFilter(result);
- if (stopSet != null)
- result = new StopFilter(result, stopSet);
- result = new SnowballFilter(result, name);
- return result;
- }
- }
+ {
+ TokenStream result = new StandardTokenizer(matchVersion, reader);
+ result = new StandardFilter(result);
+ result = new LowerCaseFilter(result);
+ if (stopSet != null)
+ result = new StopFilter(StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stopSet);
+ result = new SnowballFilter(result, name);
+ return result;
+ }
+
+ private class SavedStreams
+ {
+ internal Tokenizer source;
+ internal TokenStream result;
+ };
+
+ /** Returns a (possibly reused) {@link StandardTokenizer} filtered by a
+ * {@link StandardFilter}, a {@link LowerCaseFilter},
+ * a {@link StopFilter}, and a {@link SnowballFilter} */
+
+ public override TokenStream ReusableTokenStream(String fieldName, TextReader reader)
+ {
+ if (overridesTokenStreamMethod)
+ {
+ // LUCENE-1678: force fallback to tokenStream() if we
+ // have been subclassed and that subclass overrides
+ // tokenStream but not reusableTokenStream
+ return TokenStream(fieldName, reader);
+ }
+
+ SavedStreams streams = (SavedStreams)GetPreviousTokenStream();
+ if (streams == null)
+ {
+ streams = new SavedStreams();
+ streams.source = new StandardTokenizer(matchVersion, reader);
+ streams.result = new StandardFilter(streams.source);
+ streams.result = new LowerCaseFilter(streams.result);
+ if (stopSet != null)
+ streams.result = new StopFilter(StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stopSet);
+ streams.result = new SnowballFilter(streams.result, name);
+ SetPreviousTokenStream(streams);
+ }
+ else
+ {
+ streams.source.Reset(reader);
+ }
+ return streams.result;
+ }
+ }
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballFilter.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballFilter.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballFilter.cs Tue Feb 28 22:43:08 2012
@@ -16,6 +16,7 @@
*/
using System;
+using Lucene.Net.Analysis.Tokenattributes;
using Token = Lucene.Net.Analysis.Token;
using TokenFilter = Lucene.Net.Analysis.TokenFilter;
using TokenStream = Lucene.Net.Analysis.TokenStream;
@@ -32,54 +33,60 @@ namespace Lucene.Net.Analysis.Snowball
/// <see cref="EnglishStemmer"/> is named "English".
/// </summary>
- public class SnowballFilter : TokenFilter
+ public sealed class SnowballFilter : TokenFilter
{
private static readonly System.Object[] EMPTY_ARGS = new System.Object[0];
private SnowballProgram stemmer;
- private System.Reflection.MethodInfo stemMethod;
-
+ private TermAttribute termAtt;
+ //private System.Reflection.MethodInfo stemMethod;
+
+ public SnowballFilter(TokenStream input, SnowballProgram stemmer)
+ : base(input)
+ {
+ this.stemmer = stemmer;
+ termAtt = AddAttribute<TermAttribute>();
+ }
+
/// <summary>Construct the named stemming filter.
///
/// </summary>
- /// <param name="in_Renamed">the input tokens to stem
+ /// <param name="input">the input tokens to stem
/// </param>
/// <param name="name">the name of a stemmer
/// </param>
- public SnowballFilter(TokenStream in_Renamed, System.String name) : base(in_Renamed)
+ public SnowballFilter(TokenStream input, System.String name) : base(input)
{
try
{
System.Type stemClass = System.Type.GetType("SF.Snowball.Ext." + name + "Stemmer");
stemmer = (SnowballProgram) System.Activator.CreateInstance(stemClass);
- // why doesn't the SnowballProgram class have an (abstract?) stem method?
- stemMethod = stemClass.GetMethod("Stem", (new System.Type[0] == null) ? new System.Type[0] : (System.Type[]) new System.Type[0]);
}
catch (System.Exception e)
{
throw new System.SystemException(e.ToString());
}
+ termAtt = AddAttribute<TermAttribute>();
}
/// <summary>Returns the next input Token, after being stemmed </summary>
- public override Token Next()
+ public sealed override bool IncrementToken()
{
- Token token = input.Next();
- if (token == null)
- return null;
- stemmer.SetCurrent(token.TermText());
- try
- {
- stemMethod.Invoke(stemmer, (System.Object[]) EMPTY_ARGS);
- }
- catch (System.Exception e)
- {
- throw new System.SystemException(e.ToString());
- }
-
- Token newToken = new Token(stemmer.GetCurrent(), token.StartOffset(), token.EndOffset(), token.Type());
- newToken.SetPositionIncrement(token.GetPositionIncrement());
- return newToken;
+ if (input.IncrementToken())
+ {
+ String originalTerm = termAtt.Term();
+ stemmer.SetCurrent(originalTerm);
+ stemmer.Stem();
+ String finalTerm = stemmer.GetCurrent();
+ // Don't bother updating, if it is unchanged.
+ if (!originalTerm.Equals(finalTerm))
+ termAtt.SetTermBuffer(finalTerm);
+ return true;
+ }
+ else
+ {
+ return false;
+ }
}
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/DanishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/DanishStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/DanishStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/DanishStemmer.cs Tue Feb 28 22:43:08 2012
@@ -31,7 +31,7 @@ namespace SF.Snowball.Ext
}
private void InitBlock()
{
- a_0 = new Among[]{new Among("hed", - 1, 1, "", this), new Among("ethed", 0, 1, "", this), new Among("ered", - 1, 1, "", this), new Among("e", - 1, 1, "", this), new Among("erede", 3, 1, "", this), new Among("ende", 3, 1, "", this), new Among("erende", 5, 1, "", this), new Among("ene", 3, 1, "", this), new Among("erne", 3, 1, "", this), new Among("ere", 3, 1, "", this), new Among("en", - 1, 1, "", this), new Among("heden", 10, 1, "", this), new Among("eren", 10, 1, "", this), new Among("er", - 1, 1, "", this), new Among("heder", 13, 1, "", this), new Among("erer", 13, 1, "", this), new Among("s", - 1, 2, "", this), new Among("heds", 16, 1, "", this), new Among("es", 16, 1, "", this), new Among("endes", 18, 1, "", this), new Among("erendes", 19, 1, "", this), new Among("enes", 18, 1, "", this), new Among("ernes", 18, 1, "", this), new Among("eres", 18, 1, "", this), new Among("ens", 16, 1, "", this), new Among("hedens", 24, 1, "", this), new Among("erens", 24, 1, "", this),
new Among("ers", 16, 1, "", this), new Among("ets", 16, 1, "", this), new Among("erets", 28, 1, "", this), new Among("et", - 1, 1, "", this), new Among("eret", 30, 1, "", this)};
+ a_0 = new Among[] { new Among("hed", -1, 1, "", this), new Among("ethed", 0, 1, "", this), new Among("ered", -1, 1, "", this), new Among("e", -1, 1, "", this), new Among("erede", 3, 1, "", this), new Among("ende", 3, 1, "", this), new Among("erende", 5, 1, "", this), new Among("ene", 3, 1, "", this), new Among("erne", 3, 1, "", this), new Among("ere", 3, 1, "", this), new Among("en", -1, 1, "", this), new Among("heden", 10, 1, "", this), new Among("eren", 10, 1, "", this), new Among("er", -1, 1, "", this), new Among("heder", 13, 1, "", this), new Among("erer", 13, 1, "", this), new Among("s", -1, 2, "", this), new Among("heds", 16, 1, "", this), new Among("es", 16, 1, "", this), new Among("endes", 18, 1, "", this), new Among("erendes", 19, 1, "", this), new Among("enes", 18, 1, "", this), new Among("ernes", 18, 1, "", this), new Among("eres", 18, 1, "", this), new Among("ens", 16, 1, "", this), new Among("hedens", 24, 1, "", this), new Among("erens", 24, 1, "", t
his), new Among("ers", 16, 1, "", this), new Among("ets", 16, 1, "", this), new Among("erets", 28, 1, "", this), new Among("et", -1, 1, "", this), new Among("eret", 30, 1, "", this) };
a_1 = new Among[]{new Among("gd", - 1, - 1, "", this), new Among("dt", - 1, - 1, "", this), new Among("gt", - 1, - 1, "", this), new Among("kt", - 1, - 1, "", this)};
a_2 = new Among[]{new Among("ig", - 1, 1, "", this), new Among("lig", 0, 1, "", this), new Among("elig", 1, 1, "", this), new Among("els", - 1, 1, "", this), new Among("l\u00F8st", - 1, 2, "", this)};
}
@@ -363,7 +363,7 @@ lab1_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/DutchStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/DutchStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/DutchStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/DutchStemmer.cs Tue Feb 28 22:43:08 2012
@@ -946,7 +946,7 @@ lab9_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/EnglishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/EnglishStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/EnglishStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/EnglishStemmer.cs Tue Feb 28 22:43:08 2012
@@ -1244,7 +1244,7 @@ replab0_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/FinnishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/FinnishStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/FinnishStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/FinnishStemmer.cs Tue Feb 28 22:43:08 2012
@@ -982,7 +982,7 @@ golab6_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/FrenchStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/FrenchStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/FrenchStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/FrenchStemmer.cs Tue Feb 28 22:43:08 2012
@@ -1490,7 +1490,7 @@ lab16_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/German2Stemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/German2Stemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/German2Stemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/German2Stemmer.cs Tue Feb 28 22:43:08 2012
@@ -811,7 +811,7 @@ lab2_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/GermanStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/GermanStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/GermanStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/GermanStemmer.cs Tue Feb 28 22:43:08 2012
@@ -783,7 +783,7 @@ lab2_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/HungarianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/HungarianStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/HungarianStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/HungarianStemmer.cs Tue Feb 28 22:43:08 2012
@@ -1125,7 +1125,7 @@ namespace SF.Snowball.Ext
return true;
}
- public bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/ItalianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/ItalianStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/ItalianStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/ItalianStemmer.cs Tue Feb 28 22:43:08 2012
@@ -1124,7 +1124,7 @@ lab1_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/KpStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/KpStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/KpStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/KpStemmer.cs Tue Feb 28 22:43:08 2012
@@ -2169,7 +2169,7 @@ lab1_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/LovinsStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/LovinsStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/LovinsStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/LovinsStemmer.cs Tue Feb 28 22:43:08 2012
@@ -1876,7 +1876,7 @@ lab7_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/NorwegianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/NorwegianStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/NorwegianStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/NorwegianStemmer.cs Tue Feb 28 22:43:08 2012
@@ -277,7 +277,7 @@ lab4_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/PorterStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/PorterStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/PorterStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/PorterStemmer.cs Tue Feb 28 22:43:08 2012
@@ -653,7 +653,7 @@ lab0_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/PortugueseStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/PortugueseStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/PortugueseStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/PortugueseStemmer.cs Tue Feb 28 22:43:08 2012
@@ -1051,7 +1051,7 @@ namespace SF.Snowball.Ext
return true;
}
- public bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/RomanianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/RomanianStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/RomanianStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/RomanianStemmer.cs Tue Feb 28 22:43:08 2012
@@ -976,7 +976,7 @@ namespace SF.Snowball.Ext
return true;
}
- public bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/RussianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/RussianStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/RussianStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/RussianStemmer.cs Tue Feb 28 22:43:08 2012
@@ -581,7 +581,7 @@ lab3_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/SpanishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/SpanishStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/SpanishStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/SpanishStemmer.cs Tue Feb 28 22:43:08 2012
@@ -1104,7 +1104,7 @@ lab5_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
@@ -1147,7 +1147,8 @@ lab1_brk: ;
cursor = limit - v_2;
// do, line 217
- v_3 = limit - cursor;
do
+ v_3 = limit - cursor;
+ do
{
// (, line 217
// or, line 217
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/SwedishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/SwedishStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/SwedishStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/SwedishStemmer.cs Tue Feb 28 22:43:08 2012
@@ -281,7 +281,7 @@ lab4_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/TurkishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/TurkishStemmer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/TurkishStemmer.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/Ext/TurkishStemmer.cs Tue Feb 28 22:43:08 2012
@@ -3052,7 +3052,7 @@ namespace SF.Snowball.Ext
return true;
}
- public bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/SnowballProgram.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/SnowballProgram.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/SnowballProgram.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Snowball/SF/Snowball/SnowballProgram.cs Tue Feb 28 22:43:08 2012
@@ -15,35 +15,52 @@
* limitations under the License.
*/
using System;
+using System.Text;
+
namespace SF.Snowball
{
-
- public class SnowballProgram
+ /// <summary>
+ /// This is the rev 500 of the snowball SVN trunk,
+ /// but modified:
+ /// made abstract and introduced abstract method stem to avoid expensive reflection in filter class
+ /// </summary>
+ public abstract class SnowballProgram
{
- /// <summary> Get the current string.</summary>
- virtual public System.String GetCurrent()
- {
- return current.ToString();
- }
protected internal SnowballProgram()
{
current = new System.Text.StringBuilder();
SetCurrent("");
}
-
+
+ public abstract bool Stem();
+
/// <summary> Set the current string.</summary>
- public virtual void SetCurrent(System.String value_Renamed)
+ public virtual void SetCurrent(System.String value)
{
//// current.Replace(current.ToString(0, current.Length - 0), value_Renamed, 0, current.Length - 0);
current.Remove(0, current.Length);
- current.Append(value_Renamed);
+ current.Append(value);
cursor = 0;
limit = current.Length;
limit_backward = 0;
bra = cursor;
ket = limit;
}
-
+
+ /// <summary> Get the current string.</summary>
+ virtual public System.String GetCurrent()
+ {
+ string result = current.ToString();
+ // Make a new StringBuffer. If we reuse the old one, and a user of
+ // the library keeps a reference to the buffer returned (for example,
+ // by converting it to a String in a way which doesn't force a copy),
+ // the buffer size will not decrease, and we will risk wasting a large
+ // amount of memory.
+ // Thanks to Wolfram Esser for spotting this problem.
+ current = new StringBuilder();
+ return result;
+ }
+
// current string
protected internal System.Text.StringBuilder current;
@@ -464,7 +481,7 @@ namespace SF.Snowball
int len = ket - bra;
//// s.Replace(s.ToString(0, s.Length - 0), current.ToString(bra, ket), 0, s.Length - 0);
s.Remove(0, s.Length);
- s.Append(current.ToString(bra, ket));
+ s.Append(current.ToString(bra, len));
return s;
}
Modified: incubator/lucene.net/trunk/src/contrib/Spatial/Tier/Projectors/CartesianTierPlotter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Spatial/Tier/Projectors/CartesianTierPlotter.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Spatial/Tier/Projectors/CartesianTierPlotter.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Spatial/Tier/Projectors/CartesianTierPlotter.cs Tue Feb 28 22:43:08 2012
@@ -22,7 +22,7 @@ namespace Lucene.Net.Spatial.Tier.Projec
public class CartesianTierPlotter
{
public const String DefaltFieldPrefix = "_tier_";
- public static double EARTH_CIRC_MILES = 28892.0d;
+ public static double EARTH_CIRC_MILES = 28892.0d;
private readonly int _tierLevel;
private int _tierLength;
@@ -72,19 +72,18 @@ namespace Lucene.Net.Spatial.Tier.Projec
/// <returns></returns>
public double GetTierBoxId(double latitude, double longitude)
{
- double[] coords = _projector.Coords(latitude, longitude);
- double[] ranges = _projector.Range();
+ var coords = _projector.Coords(latitude, longitude);
+ var ranges = _projector.Range();
- double id = GetBoxCoord(coords[0], ranges[0]) + (GetBoxCoord(coords[1], ranges[1]) / TierVerticalPosDivider);
- return id;
+ double id = GetBoxCoord(coords[0], ranges[0]) + (GetBoxCoord(coords[1], ranges[1]) / TierVerticalPosDivider);
+ return id;
}
private double GetBoxCoord(double coord, double range)
{
- return Math.Floor(coord*(this._tierLength/range));
+ return Math.Floor(coord * (_tierLength/range));
}
-
/// <summary>
/// Get the string name representing current tier _localTier<tiedId>
/// </summary>
@@ -116,8 +115,8 @@ namespace Lucene.Net.Spatial.Tier.Projec
/// <returns></returns>
public int BestFit(double range)
{
- double times = EARTH_CIRC_MILES / (2.0d * range);
- int bestFit = (int)Math.Ceiling(Math.Log(times, 2));
+ var times = EARTH_CIRC_MILES / (2.0d * range);
+ var bestFit = (int) Math.Ceiling(Math.Log(times, 2));
if (bestFit > 15)
{
Modified: incubator/lucene.net/trunk/src/contrib/Spatial/Tier/Projectors/SinusoidalProjector.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/Spatial/Tier/Projectors/SinusoidalProjector.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/Spatial/Tier/Projectors/SinusoidalProjector.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/Spatial/Tier/Projectors/SinusoidalProjector.cs Tue Feb 28 22:43:08 2012
@@ -26,7 +26,8 @@ namespace Lucene.Net.Spatial.Tier.Projec
/// </summary>
public class SinusoidalProjector : IProjector
{
- public static double LATITUDE_RANGE = Math.PI;
+
+ public static double LATITUDE_RANGE = Math.PI;
public static double LONGITUDE_RANGE = 2*Math.PI;
public string CoordsAsString(double latitude, double longitude)
@@ -34,6 +35,12 @@ namespace Lucene.Net.Spatial.Tier.Projec
return null;
}
+ public double[] Range()
+ {
+ double[] ranges = { LATITUDE_RANGE, LONGITUDE_RANGE };
+ return ranges;
+ }
+
public double[] Coords(double latitude, double longitude)
{
double rlat = MathHelper.ToRadians(latitude);
@@ -42,11 +49,5 @@ namespace Lucene.Net.Spatial.Tier.Projec
double[] r = {nlat, rlong};
return r;
}
-
- public double[] Range()
- {
- double[] ranges = {LATITUDE_RANGE, LONGITUDE_RANGE};
- return ranges;
- }
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/src/contrib/SpellChecker/Contrib.SpellChecker.csproj
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/SpellChecker/Contrib.SpellChecker.csproj?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/SpellChecker/Contrib.SpellChecker.csproj (original)
+++ incubator/lucene.net/trunk/src/contrib/SpellChecker/Contrib.SpellChecker.csproj Tue Feb 28 22:43:08 2012
@@ -19,7 +19,6 @@
under the License.
-->
-
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<ProjectType>Local</ProjectType>
@@ -108,7 +107,7 @@
<Compile Include="AssemblyInfo.cs">
<SubType>Code</SubType>
</Compile>
- <Compile Include="Spell\Dictionary.cs">
+ <Compile Include="Spell\IDictionary.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="Spell\JaroWinklerDistance.cs" />