You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by cc...@apache.org on 2012/02/28 23:43:28 UTC
[Lucene.Net] svn commit: r1294875 [9/45] - in /incubator/lucene.net/trunk: ./ build/
build/vs2010/contrib/ build/vs2010/test/ doc/ src/ src/contrib/Analyzers/
src/contrib/Analyzers/AR/ src/contrib/Analyzers/BR/
src/contrib/Analyzers/CJK/ src/contrib/Analyzers/Cn/ s...
Modified: incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/JaroWinklerDistance.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/JaroWinklerDistance.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/JaroWinklerDistance.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/JaroWinklerDistance.cs Tue Feb 28 22:43:08 2012
@@ -15,19 +15,22 @@
* limitations under the License.
*/
-using System;
-using System.Collections.Generic;
-using System.Text;
+
+using System.Linq;
namespace SpellChecker.Net.Search.Spell
{
+
+ using System;
+
public class JaroWinklerDistance : StringDistance
{
private float threshold = 0.7f;
- private int[] Matches(String s1, String s2)
+ private static int[] Matches(String s1, String s2)
{
String Max, Min;
+
if (s1.Length > s2.Length)
{
Max = s1;
@@ -38,29 +41,34 @@ namespace SpellChecker.Net.Search.Spell
Max = s2;
Min = s1;
}
- int range = Math.Max(Max.Length / 2 - 1, 0);
- int[] matchIndexes = new int[Min.Length];
- for (int i = 0; i < matchIndexes.Length; i++)
+
+ var range = Math.Max(Max.Length / 2 - 1, 0);
+ var matchIndexes = new int[Min.Length];
+
+ for (var i = 0; i < matchIndexes.Length; i++)
matchIndexes[i] = -1;
- bool[] matchFlags = new bool[Max.Length];
- int matches = 0;
- for (int mi = 0; mi < Min.Length; mi++)
- {
- char c1 = Min[mi];
- for (int xi = Math.Max(mi - range, 0), xn = Math.Min(mi + range + 1, Max
- .Length); xi < xn; xi++)
- {
- if (!matchFlags[xi] && c1 == Max[xi])
- {
- matchIndexes[mi] = xi;
- matchFlags[xi] = true;
- matches++;
- break;
- }
+
+ var matchFlags = new bool[Max.Length];
+ var matches = 0;
+
+ for (var mi = 0; mi < Min.Length; mi++)
+ {
+ var c1 = Min[mi];
+ for (int xi = Math.Max(mi - range, 0),
+ xn = Math.Min(mi + range + 1, Max.Length); xi < xn; xi++)
+ {
+ if (matchFlags[xi] || c1 != Max[xi]) continue;
+
+ matchIndexes[mi] = xi;
+ matchFlags[xi] = true;
+ matches++;
+ break;
}
}
- char[] ms1 = new char[matches];
- char[] ms2 = new char[matches];
+
+ var ms1 = new char[matches];
+ var ms2 = new char[matches];
+
for (int i = 0, si = 0; i < Min.Length; i++)
{
if (matchIndexes[i] != -1)
@@ -69,6 +77,7 @@ namespace SpellChecker.Net.Search.Spell
si++;
}
}
+
for (int i = 0, si = 0; i < Max.Length; i++)
{
if (matchFlags[i])
@@ -77,16 +86,11 @@ namespace SpellChecker.Net.Search.Spell
si++;
}
}
- int transpositions = 0;
- for (int mi = 0; mi < ms1.Length; mi++)
- {
- if (ms1[mi] != ms2[mi])
- {
- transpositions++;
- }
- }
- int prefix = 0;
- for (int mi = 0; mi < Min.Length; mi++)
+
+ var transpositions = ms1.Where((t, mi) => t != ms2[mi]).Count();
+
+ var prefix = 0;
+ for (var mi = 0; mi < Min.Length; mi++)
{
if (s1[mi] == s2[mi])
{
@@ -97,25 +101,25 @@ namespace SpellChecker.Net.Search.Spell
break;
}
}
+
return new int[] { matches, transpositions / 2, prefix, Max.Length };
}
public float GetDistance(String s1, String s2)
{
- int[] mtp = Matches(s1, s2);
- float m = (float)mtp[0];
+ var mtp = Matches(s1, s2);
+ var m = (float)mtp[0];
+
if (m == 0)
- {
return 0f;
- }
+
float j = ((m / s1.Length + m / s2.Length + (m - mtp[1]) / m)) / 3;
- float jw = j < GetThreshold() ? j : j + Math.Min(0.1f, 1f / mtp[3]) * mtp[2]
- * (1 - j);
+ float jw = j < GetThreshold() ? j : j + Math.Min(0.1f, 1f / mtp[3]) * mtp[2] * (1 - j);
return jw;
}
/// <summary>
- ///Sets the threshold used to deterMine when Winkler bonus should be used.
+ /// Sets the threshold used to deterMine when Winkler bonus should be used.
/// Set to a negative value to get the Jaro distance.
/// </summary>
/// <param name="threshold">the new value of the threshold</param>
Modified: incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/LuceneDictionary.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/LuceneDictionary.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/LuceneDictionary.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/LuceneDictionary.cs Tue Feb 28 22:43:08 2012
@@ -15,17 +15,17 @@
* limitations under the License.
*/
-using System;
-using IndexReader = Lucene.Net.Index.IndexReader;
-using TermEnum = Lucene.Net.Index.TermEnum;
-using Term = Lucene.Net.Index.Term;
-
namespace SpellChecker.Net.Search.Spell
{
- /// <summary> Lucene Dictionary
- ///
+ using System;
+ using IndexReader = Lucene.Net.Index.IndexReader;
+ using TermEnum = Lucene.Net.Index.TermEnum;
+ using Term = Lucene.Net.Index.Term;
+
+ /// <summary>
+ /// Lucene Dictionary
/// </summary>
- public class LuceneDictionary : Dictionary
+ public class LuceneDictionary : IDictionary
{
internal IndexReader reader;
internal System.String field;
@@ -49,11 +49,11 @@ namespace SpellChecker.Net.Search.Spell
internal sealed class LuceneIterator : System.Collections.IEnumerator
{
- private TermEnum termEnum;
+ private readonly TermEnum termEnum;
private Term actualTerm;
private bool hasNextCalled;
- private LuceneDictionary enclosingInstance;
+ private readonly LuceneDictionary enclosingInstance;
public LuceneIterator(LuceneDictionary enclosingInstance)
{
Modified: incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/PlainTextDictionary.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/PlainTextDictionary.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/PlainTextDictionary.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/PlainTextDictionary.cs Tue Feb 28 22:43:08 2012
@@ -30,7 +30,7 @@ namespace SpellChecker.Net.Search.Spell
/// </summary>
/// <author> Nicolas Maisonneuve
/// </author>
- public class PlainTextDictionary : Dictionary
+ public class PlainTextDictionary : IDictionary
{
virtual public System.Collections.IEnumerator GetWordsIterator()
{
Modified: incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/SpellChecker.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/SpellChecker.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/SpellChecker.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/SpellChecker.cs Tue Feb 28 22:43:08 2012
@@ -15,27 +15,27 @@
* limitations under the License.
*/
-using System;
-
-using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
-using Document = Lucene.Net.Documents.Document;
-using Field = Lucene.Net.Documents.Field;
-using IndexReader = Lucene.Net.Index.IndexReader;
-using IndexWriter = Lucene.Net.Index.IndexWriter;
-using Term = Lucene.Net.Index.Term;
-using BooleanClause = Lucene.Net.Search.BooleanClause;
-using BooleanQuery = Lucene.Net.Search.BooleanQuery;
-using Hits = Lucene.Net.Search.Hits;
-using IndexSearcher = Lucene.Net.Search.IndexSearcher;
-using Query = Lucene.Net.Search.Query;
-using TermQuery = Lucene.Net.Search.TermQuery;
-using Directory = Lucene.Net.Store.Directory;
-using SpellChecker.Net.Search.Spell;
-using Lucene.Net.Store;
-using Lucene.Net.Search;
namespace SpellChecker.Net.Search.Spell
{
+ using System;
+
+ using Lucene.Net.Search;
+ using Lucene.Net.Store;
+ using BooleanClause = Lucene.Net.Search.BooleanClause;
+ using BooleanQuery = Lucene.Net.Search.BooleanQuery;
+ using Directory = Lucene.Net.Store.Directory;
+ using Document = Lucene.Net.Documents.Document;
+ using Field = Lucene.Net.Documents.Field;
+ using IndexReader = Lucene.Net.Index.IndexReader;
+ using IndexSearcher = Lucene.Net.Search.IndexSearcher;
+ using IndexWriter = Lucene.Net.Index.IndexWriter;
+ using Query = Lucene.Net.Search.Query;
+ using Term = Lucene.Net.Index.Term;
+ using TermQuery = Lucene.Net.Search.TermQuery;
+ using WhitespaceAnalyzer = Lucene.Net.Analysis.WhitespaceAnalyzer;
+
+
/// <summary> <p>
/// Spell Checker class (Main class) <br/>
/// (initially inspired by the David Spencer code).
@@ -67,17 +67,16 @@ namespace SpellChecker.Net.Search.Spell
internal Directory spellindex;
/// <summary> Boost value for start and end grams</summary>
- private float bStart = 2.0f;
- private float bEnd = 1.0f;
+ private const float bStart = 2.0f;
+ private const float bEnd = 1.0f;
- //private IndexReader reader;
// don't use this searcher directly - see #swapSearcher()
private IndexSearcher searcher;
/// <summary>
/// this locks all modifications to the current searcher.
/// </summary>
- private static System.Object searcherLock = new System.Object();
+ private static readonly System.Object searcherLock = new System.Object();
/*
* this lock synchronizes all possible modifications to the
@@ -85,7 +84,7 @@ namespace SpellChecker.Net.Search.Spell
* the same index concurrently. Note: Do not acquire the searcher lock
* before acquiring this lock!
*/
- private static System.Object modifyCurrentIndexLock = new System.Object();
+ private static readonly System.Object modifyCurrentIndexLock = new System.Object();
private volatile bool closed = false;
internal float minScore = 0.5f; //LUCENENET-359 Spellchecker accuracy gets overwritten
@@ -96,11 +95,11 @@ namespace SpellChecker.Net.Search.Spell
/// Use the given directory as a spell checker index. The directory
/// is created if it doesn't exist yet.
/// </summary>
- /// <param name="gramIndex">the spell index directory</param>
+ /// <param name="spellIndex">the spell index directory</param>
/// <param name="sd">the <see cref="StringDistance"/> measurement to use </param>
- public SpellChecker(Directory gramIndex, StringDistance sd)
+ public SpellChecker(Directory spellIndex, StringDistance sd)
{
- this.SetSpellIndex(gramIndex);
+ this.SetSpellIndex(spellIndex);
this.setStringDistance(sd);
}
@@ -109,9 +108,9 @@ namespace SpellChecker.Net.Search.Spell
/// <see cref="LevenshteinDistance"/> as the default <see cref="StringDistance"/>. The
/// directory is created if it doesn't exist yet.
/// </summary>
- /// <param name="gramIndex">the spell index directory</param>
- public SpellChecker(Directory gramIndex)
- : this(gramIndex, new LevenshteinDistance())
+ /// <param name="spellIndex">the spell index directory</param>
+ public SpellChecker(Directory spellIndex)
+ : this(spellIndex, new LevenshteinDistance())
{ }
/// <summary>
@@ -131,7 +130,7 @@ namespace SpellChecker.Net.Search.Spell
EnsureOpen();
if (!IndexReader.IndexExists(spellIndexDir))
{
- IndexWriter writer = new IndexWriter(spellIndexDir, null, true,
+ var writer = new IndexWriter(spellIndexDir, null, true,
IndexWriter.MaxFieldLength.UNLIMITED);
writer.Close();
}
@@ -253,7 +252,7 @@ namespace SpellChecker.Net.Search.Spell
int maxHits = 10 * numSug;
// System.out.println("Q: " + query);
- ScoreDoc[] hits = indexSearcher.Search(query, null, maxHits).scoreDocs;
+ ScoreDoc[] hits = indexSearcher.Search(query, null, maxHits).ScoreDocs;
// System.out.println("HITS: " + hits.length());
SuggestWordQueue sugQueue = new SuggestWordQueue(numSug);
@@ -390,7 +389,7 @@ namespace SpellChecker.Net.Search.Spell
/// <param name="ramMB">the max amount or memory in MB to use</param>
/// <throws> IOException </throws>
/// <throws>AlreadyClosedException if the Spellchecker is already closed</throws>
- public virtual void IndexDictionary(Dictionary dict, int mergeFactor, int ramMB)
+ public virtual void IndexDictionary(IDictionary dict, int mergeFactor, int ramMB)
{
lock (modifyCurrentIndexLock)
{
@@ -431,10 +430,10 @@ namespace SpellChecker.Net.Search.Spell
}
/// <summary>
- /// Indexes the data from the given <see cref="Dictionary"/>.
+ /// Indexes the data from the given <see cref="IDictionary"/>.
/// </summary>
/// <param name="dict">dict the dictionary to index</param>
- public void IndexDictionary(Dictionary dict)
+ public void IndexDictionary(IDictionary dict)
{
IndexDictionary(dict, 300, 10);
}
Modified: incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/StringDistance.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/StringDistance.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/StringDistance.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/StringDistance.cs Tue Feb 28 22:43:08 2012
@@ -15,12 +15,11 @@
* limitations under the License.
*/
-using System;
-using System.Collections.Generic;
-using System.Text;
namespace SpellChecker.Net.Search.Spell
{
+ using System;
+
/// <summary>
/// Interface for string distances.
/// </summary>
Modified: incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/SuggestWordQueue.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/SuggestWordQueue.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/SuggestWordQueue.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/SuggestWordQueue.cs Tue Feb 28 22:43:08 2012
@@ -15,13 +15,11 @@
* limitations under the License.
*/
-using System;
-using PriorityQueue = Lucene.Net.Util.PriorityQueue;
namespace SpellChecker.Net.Search.Spell
{
-
-
+ using PriorityQueue = Lucene.Net.Util.PriorityQueue<SuggestWord>;
+
sealed class SuggestWordQueue : PriorityQueue
{
@@ -30,11 +28,9 @@ namespace SpellChecker.Net.Search.Spell
Initialize(size);
}
- override public bool LessThan(System.Object a, System.Object b)
+ override public bool LessThan(SuggestWord a, SuggestWord b)
{
- SuggestWord wa = (SuggestWord) a;
- SuggestWord wb = (SuggestWord) b;
- int val = wa.CompareTo(wb);
+ var val = a.CompareTo(b);
return val < 0;
}
}
Modified: incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/TRStringDistance.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/TRStringDistance.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/TRStringDistance.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/SpellChecker/Spell/TRStringDistance.cs Tue Feb 28 22:43:08 2012
@@ -15,8 +15,6 @@
* limitations under the License.
*/
-using System;
-using SpellChecker.Net.Search.Spell;
namespace SpellChecker.Net.Search.Spell
{
@@ -46,8 +44,7 @@ namespace SpellChecker.Net.Search.Spell
public int GetDistance(System.String other)
{
int[][] d; // matrix
- int cost; // cost
-
+
// Step 1
char[] ta = other.ToCharArray();
int m = ta.Length;
@@ -85,19 +82,8 @@ namespace SpellChecker.Net.Search.Spell
char t_j = ta[j - 1];
// Step 5
-
- if (s_i == t_j)
- {
- // same
- cost = 0;
- }
- else
- {
- // not a match
- cost = 1;
-
- // Step 6
- }
+
+ int cost = s_i == t_j ? 0 : 1;
d[i][j] = Min3(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost);
}
}
Modified: incubator/lucene.net/trunk/src/contrib/WordNet/SynExpand/Contrib.WordNet.SynExpand.csproj
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/WordNet/SynExpand/Contrib.WordNet.SynExpand.csproj?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/WordNet/SynExpand/Contrib.WordNet.SynExpand.csproj (original)
+++ incubator/lucene.net/trunk/src/contrib/WordNet/SynExpand/Contrib.WordNet.SynExpand.csproj Tue Feb 28 22:43:08 2012
@@ -19,7 +19,6 @@
under the License.
-->
-
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="Build">
<PropertyGroup>
<ProjectType>Local</ProjectType>
@@ -116,12 +115,6 @@
<Content Include="App.ico" />
</ItemGroup>
<ItemGroup>
- <ProjectReference Include="..\..\..\core\Lucene.Net.csproj">
- <Project>{5D4AD9BE-1FFB-41AB-9943-25737971BF57}</Project>
- <Name>Lucene.Net</Name>
- </ProjectReference>
- </ItemGroup>
- <ItemGroup>
<BootstrapperPackage Include=".NETFramework,Version=v4.0">
<Visible>False</Visible>
<ProductName>Microsoft .NET Framework 4 %28x86 and x64%29</ProductName>
@@ -143,6 +136,12 @@
<Install>true</Install>
</BootstrapperPackage>
</ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\..\..\core\Lucene.Net.csproj">
+ <Project>{5D4AD9BE-1FFB-41AB-9943-25737971BF57}</Project>
+ <Name>Lucene.Net</Name>
+ </ProjectReference>
+ </ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<PropertyGroup>
<PreBuildEvent />
Modified: incubator/lucene.net/trunk/src/contrib/WordNet/SynExpand/SynExpand.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/WordNet/SynExpand/SynExpand.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/WordNet/SynExpand/SynExpand.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/WordNet/SynExpand/SynExpand.cs Tue Feb 28 22:43:08 2012
@@ -16,152 +16,179 @@
*/
using System;
-
-using Lucene.Net.Store;
-using Lucene.Net.Search;
-using Lucene.Net.Index;
-using Lucene.Net.Documents;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
namespace WorldNet.Net
{
-
-
- /// <summary> Expand a query by looking up synonyms for every term.
- /// You need to invoke <see cref="Syns2Index"/> first to build the synonym index.
- ///
- /// </summary>
- /// <seealso cref="Syns2Index">
- /// </seealso>
- public sealed class SynExpand
- {
-
- /// <summary> Test driver for synonym expansion.
- /// Uses boost factor of 0.9 for illustrative purposes.
- ///
- /// If you pass in the query "big dog" then it prints out:
- ///
- /// <pre>
- /// Query: big adult^0.9 bad^0.9 bighearted^0.9 boastful^0.9 boastfully^0.9 bounteous^0.9 bountiful^0.9 braggy^0.9 crowing^0.9 freehanded^0.9 giving^0.9 grown^0.9 grownup^0.9 handsome^0.9 large^0.9 liberal^0.9 magnanimous^0.9 momentous^0.9 openhanded^0.9 prominent^0.9 swelled^0.9 vainglorious^0.9 vauntingly^0.9
- /// dog andiron^0.9 blackguard^0.9 bounder^0.9 cad^0.9 chase^0.9 click^0.9 detent^0.9 dogtooth^0.9 firedog^0.9 frank^0.9 frankfurter^0.9 frump^0.9 heel^0.9 hotdog^0.9 hound^0.9 pawl^0.9 tag^0.9 tail^0.9 track^0.9 trail^0.9 weenie^0.9 wiener^0.9 wienerwurst^0.9
- /// </pre>
- /// </summary>
- [STAThread]
- public static void Main(System.String[] args)
- {
- if (args.Length != 2)
- {
- System.Console.Out.WriteLine(typeof(SynExpand) + " <index path> <query>");
+
+
+ /// <summary> Expand a query by looking up synonyms for every term.
+ /// You need to invoke <see cref="Syns2Index"/> first to build the synonym index.
+ ///
+ /// </summary>
+ /// <seealso cref="Syns2Index" />
+ public sealed class SynExpand
+ {
+ static List<String> already;
+ private static BooleanQuery tmp;
+
+ /// <summary> Test driver for synonym expansion.
+ /// Uses boost factor of 0.9 for illustrative purposes.
+ ///
+ /// If you pass in the query "big dog" then it prints out:
+ ///
+ /// <pre>
+ /// Query: big adult^0.9 bad^0.9 bighearted^0.9 boastful^0.9 boastfully^0.9 bounteous^0.9 bountiful^0.9 braggy^0.9 crowing^0.9 freehanded^0.9 giving^0.9 grown^0.9 grownup^0.9 handsome^0.9 large^0.9 liberal^0.9 magnanimous^0.9 momentous^0.9 openhanded^0.9 prominent^0.9 swelled^0.9 vainglorious^0.9 vauntingly^0.9
+ /// dog andiron^0.9 blackguard^0.9 bounder^0.9 cad^0.9 chase^0.9 click^0.9 detent^0.9 dogtooth^0.9 firedog^0.9 frank^0.9 frankfurter^0.9 frump^0.9 heel^0.9 hotdog^0.9 hound^0.9 pawl^0.9 tag^0.9 tail^0.9 track^0.9 trail^0.9 weenie^0.9 wiener^0.9 wienerwurst^0.9
+ /// </pre>
+ /// </summary>
+ [STAThread]
+ public static void Main(String[] args)
+ {
+ if (args.Length != 2)
+ {
+ Console.Out.WriteLine(typeof(SynExpand) + " <index path> <query>");
return;
- }
-
- FSDirectory directory = FSDirectory.GetDirectory(args[0], false);
- IndexSearcher searcher = new IndexSearcher(directory);
-
- System.String query = args[1];
- System.String field = "contents";
-
- Query q = Expand(query, searcher, new StandardAnalyzer(), field, 0.9f);
- System.Console.Out.WriteLine("Query: " + q.ToString(field));
-
-
-
- searcher.Close();
- directory.Close();
- }
-
-
- /// <summary> Perform synonym expansion on a query.
- ///
- /// </summary>
- /// <param name="query">users query that is assumed to not have any "special" query syntax, thus it should be just normal words, so "big dog" makes sense, but a query like "title:foo^1.2" doesn't as this should presumably be passed directly to the default query parser.
- ///
- /// </param>
- /// <param name="syns">a opened to the Lucene index you previously created with <see cref="Syns2Index"/>. The searcher is not closed or otherwise altered.
- ///
- /// </param>
- /// <param name="a">optional analyzer used to parse the users query else <see cref="StandardAnalyzer"/> is used
- ///
- /// </param>
- /// <param name="field">optional field name to search in or null if you want the default of "contents"
- ///
- /// </param>
- /// <param name="boost">optional boost applied to synonyms else no boost is applied
- ///
- /// </param>
- /// <returns> the expanded Query
- /// </returns>
- public static Query Expand(System.String query, Searcher syns, Analyzer a, System.String field, float boost)
+ }
+
+ var directory = FSDirectory.Open(new DirectoryInfo(args[0]));
+ var searcher = new IndexSearcher(directory, true);
+
+ String query = args[1];
+ const string field = "contents";
+
+ Query q = Expand(query, searcher, new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT), field, 0.9f);
+ System.Console.Out.WriteLine("Query: " + q.ToString(field));
+
+ searcher.Close();
+ directory.Close();
+ }
+
+
+ /// <summary>
+ /// Perform synonym expansion on a query.
+ /// </summary>
+ /// <param name="query">users query that is assumed to not have any "special" query syntax, thus it should be just normal words, so "big dog" makes sense, but a query like "title:foo^1.2" doesn't as this should presumably be passed directly to the default query parser </param>
+ /// <param name="syns">a opened to the Lucene index you previously created with <see cref="Syns2Index"/>. The searcher is not closed or otherwise altered. </param>
+ /// <param name="a">optional analyzer used to parse the users query else <see cref="StandardAnalyzer"/> is used </param>
+ /// <param name="field">optional field name to search in or null if you want the default of "contents" </param>
+ /// <param name="boost">optional boost applied to synonyms else no boost is applied </param>
+ /// <returns>the expanded Query </returns>
+ public static Query Expand(String query,
+ Searcher syns,
+ Analyzer a,
+ String field,
+ float boost)
{
- System.Collections.Hashtable already = new System.Collections.Hashtable(); // avoid dups
- System.Collections.IList top = new System.Collections.ArrayList(); // needs to be separately listed..
+ already = new List<String>(); // avoid dups
+ var top = new List<String>(); // needs to be separately listed..
if (field == null)
field = "contents";
- if (a == null)
- a = new StandardAnalyzer();
+
+ if (a == null)
+ a = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT);
// [1] Parse query into separate words so that when we expand we can avoid dups
- TokenStream ts = a.TokenStream(field, new System.IO.StringReader(query));
- Lucene.Net.Analysis.Token t;
- while ((t = ts.Next()) != null)
+ var ts = a.TokenStream(field, new StringReader(query));
+ var termAtt = ts.AddAttribute<TermAttribute>();
+
+ while (ts.IncrementToken())
{
- System.String word = t.TermText();
- if (already.Contains(word) == false)
+ var word = termAtt.Term();
+
+ if (!already.Contains(word))
{
- already.Add(word, word);
+ already.Add(word);
top.Add(word);
}
}
- BooleanQuery tmp = new BooleanQuery();
+
+ tmp = new BooleanQuery();
// [2] form query
System.Collections.IEnumerator it = top.GetEnumerator();
while (it.MoveNext())
{
// [2a] add to level words in
- System.String word = (System.String) it.Current;
- TermQuery tq = new TermQuery(new Term(field, word));
+ var word = (String) it.Current;
+ var tq = new TermQuery(new Term(field, word));
tmp.Add(tq, BooleanClause.Occur.SHOULD);
-
- // [2b] add in unique synonums
- Hits hits = syns.Search(new TermQuery(new Term(Syns2Index.F_WORD, word)));
- for (int i = 0; i < hits.Length(); i++)
- {
- Document doc = hits.Doc(i);
- System.String[] values = doc.GetValues(Syns2Index.F_SYN);
- for (int j = 0; j < values.Length; j++)
- {
- System.String syn = values[j];
- if (already.Contains(syn) == false)
- // avoid dups of top level words and synonyms
- {
- already.Add(syn, syn);
- tq = new TermQuery(new Term(field, syn));
- if (boost > 0)
- // else keep normal 1.0
- tq.SetBoost(boost);
- tmp.Add(tq, BooleanClause.Occur.SHOULD);
- }
- }
- }
+
+ var c = new CollectorImpl(field, boost);
+ syns.Search(new TermQuery(new Term(Syns2Index.F_WORD, word)), c);
}
-
return tmp;
}
- }
+
+
+ /// <summary>
+ /// From project WordNet.Net.Syns2Index
+ /// </summary>
+ public class Syns2Index
+ {
+ /// <summary> </summary>
+ public const String F_SYN = "syn";
+
+ /// <summary> </summary>
+ public const String F_WORD = "word";
+ }
+
+ /// <summary>
+ /// CollectorImpl
+ /// </summary>
+ internal sealed class CollectorImpl : Collector
+ {
+ private IndexReader reader;
+ private readonly string field;
+ private readonly float boost;
+
+ public CollectorImpl(string field, float boost)
+ {
+ this.field = field;
+ this.boost = boost;
+ }
+
+ public override void SetScorer(Scorer scorer)
+ {
+ // Ignore
+ }
+
+ public override void Collect(int doc)
+ {
+ var d = reader.Document(doc);
+ var values = d.GetValues(Syns2Index.F_SYN);
+ foreach (var syn in values.Where(syn => !already.Contains(syn)))
+ {
+ already.Add(syn);
+
+ var tq = new TermQuery(new Term(field, syn));
+ if (boost > 0) // else keep normal 1.0
+ tq.SetBoost(boost);
+
+ tmp.Add(tq, BooleanClause.Occur.SHOULD);
+ }
+ }
+
+ public override void SetNextReader(IndexReader reader, int docBase)
+ {
+ this.reader = reader;
+ }
+
+ public override bool AcceptsDocsOutOfOrder()
+ {
+ return true;
+ }
+ }
- /// <summary>
- /// From project WordNet.Net.Syns2Index
- /// </summary>
- public class Syns2Index
- {
- /// <summary> </summary>
- public const System.String F_SYN = "syn";
-
- /// <summary> </summary>
- public const System.String F_WORD = "word";
- }
+ }
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/src/contrib/WordNet/SynLookup/SynLookup.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/WordNet/SynLookup/SynLookup.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/WordNet/SynLookup/SynLookup.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/WordNet/SynLookup/SynLookup.cs Tue Feb 28 22:43:08 2012
@@ -16,140 +16,193 @@
*/
using System;
-
-using Lucene.Net.Store;
-using Lucene.Net.Search;
-using Lucene.Net.Index;
-using Lucene.Net.Documents;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Index;
+using Lucene.Net.Search;
+using Lucene.Net.Store;
namespace WorldNet.Net
{
-
-
/// <summary> Test program to look up synonyms.</summary>
public class SynLookup
{
-
+ static List<String> already;
+ private static BooleanQuery tmp;
+
[STAThread]
public static void Main(System.String[] args)
{
if (args.Length != 2)
{
System.Console.Out.WriteLine(typeof(SynLookup) + " <index path> <word>");
- return;
+ return;
}
- FSDirectory directory = FSDirectory.GetDirectory(args[0], false);
- IndexSearcher searcher = new IndexSearcher(directory);
-
- System.String word = args[1];
- Hits hits = searcher.Search(new TermQuery(new Term(Syns2Index.F_WORD, word)));
-
- if (hits.Length() == 0)
+ using (var directory = FSDirectory.Open(new DirectoryInfo(args[0])))
{
- System.Console.Out.WriteLine("No synonyms found for " + word);
- }
- else
- {
- System.Console.Out.WriteLine("Synonyms found for \"" + word + "\":");
- }
-
- for (int i = 0; i < hits.Length(); i++)
- {
- Document doc = hits.Doc(i);
-
- System.String[] values = doc.GetValues(Syns2Index.F_SYN);
-
- for (int j = 0; j < values.Length; j++)
+ using (var searcher = new IndexSearcher(directory, true))
{
- System.Console.Out.WriteLine(values[j]);
+
+ String word = args[1];
+ Query query = new TermQuery(new Term(Syns2Index.F_WORD, word));
+ var countingCollector = new CountingCollector();
+ searcher.Search(query, countingCollector);
+
+ if (countingCollector.numHits == 0)
+ {
+ Console.Out.WriteLine("No synonyms found for " + word);
+ }
+ else
+ {
+ Console.Out.WriteLine("Synonyms found for \"" + word + "\":");
+ }
+
+ var hits = searcher.Search(query, countingCollector.numHits).ScoreDocs;
+
+ foreach (var v in
+ hits.Select(t => searcher.Doc(t.doc)).Select(doc => doc.GetValues(Syns2Index.F_SYN)).SelectMany(values => values))
+ {
+ Console.Out.WriteLine(v);
+ }
+
}
}
-
- searcher.Close();
- directory.Close();
}
-
- /// <summary> Perform synonym expansion on a query.
- ///
+ /// <summary>
+ /// Perform synonym expansion on a query.
/// </summary>
- /// <param name="">query
- /// </param>
- /// <param name="">syns
- /// </param>
- /// <param name="">a
- /// </param>
- /// <param name="">field
- /// </param>
- /// <param name="">boost
- /// </param>
- public static Query Expand(System.String query, Searcher syns, Analyzer a, System.String field, float boost)
+ /// <param name="query">query</param>
+ /// <param name="syns">syns</param>
+ /// <param name="a">a</param>
+ /// <param name="field">field</param>
+ /// <param name="boost">boost</param>
+ public static Query Expand(String query,
+ Searcher syns,
+ Analyzer a,
+ String field,
+ float boost)
{
- System.Collections.Hashtable already = new System.Collections.Hashtable(); // avoid dups
- System.Collections.IList top = new System.Collections.ArrayList(); // needs to be separately listed..
-
- // [1] Parse query into separate words so that when we expand we can avoid dups
- TokenStream ts = a.TokenStream(field, new System.IO.StringReader(query));
- Lucene.Net.Analysis.Token t;
- while ((t = ts.Next()) != null)
+ already = new List<String>(); // avoid dups
+ var top = new List<String>(); // needs to be separately listed..
+
+ var ts = a.TokenStream(field, new StringReader(query));
+ var termAtt = ts.AddAttribute<TermAttribute>();
+
+ while (ts.IncrementToken())
{
- System.String word = t.TermText();
- if (already.Contains(word) == false)
+ var word = termAtt.Term();
+
+ if (!already.Contains(word))
{
- already.Add(word, word);
+ already.Add(word);
top.Add(word);
}
}
- BooleanQuery tmp = new BooleanQuery();
-
+
+ tmp = new BooleanQuery();
+
// [2] form query
System.Collections.IEnumerator it = top.GetEnumerator();
while (it.MoveNext())
{
// [2a] add to level words in
- System.String word = (System.String) it.Current;
- TermQuery tq = new TermQuery(new Term(field, word));
+ var word = (String)it.Current;
+ var tq = new TermQuery(new Term(field, word));
tmp.Add(tq, BooleanClause.Occur.SHOULD);
-
- // [2b] add in unique synonums
- Hits hits = syns.Search(new TermQuery(new Term(Syns2Index.F_WORD, word)));
- for (int i = 0; i < hits.Length(); i++)
- {
- Document doc = hits.Doc(i);
- System.String[] values = doc.GetValues(Syns2Index.F_SYN);
- for (int j = 0; j < values.Length; j++)
- {
- System.String syn = values[j];
- if (already.Contains(syn) == false)
- {
- already.Add(syn, syn);
- tq = new TermQuery(new Term(field, syn));
- if (boost > 0)
- // else keep normal 1.0
- tq.SetBoost(boost);
- tmp.Add(tq, BooleanClause.Occur.SHOULD);
- }
- }
- }
+
+ var c = new CollectorImpl(field, boost);
+ syns.Search(new TermQuery(new Term(Syns2Index.F_WORD, word)), c);
}
-
-
+
return tmp;
}
- }
+ internal sealed class CountingCollector : Collector
+ {
+ public int numHits;
+
+ public override void SetScorer(Scorer scorer)
+ { }
+
+ public override void Collect(int doc)
+ {
+ numHits++;
+ }
+
+ public override void SetNextReader(IndexReader reader, int docBase)
+ { }
+
+ public override bool AcceptsDocsOutOfOrder()
+ {
+ return true;
+ }
+ }
+
+ /// <summary>
+ /// CollectorImpl
+ /// </summary>
+ internal sealed class CollectorImpl : Collector
+ {
+ private IndexReader reader;
+ private readonly string field;
+ private readonly float boost;
+
+ public CollectorImpl(string field, float boost)
+ {
+ this.field = field;
+ this.boost = boost;
+ }
+
+ public override void SetScorer(Scorer scorer)
+ {
+ // Ignore
+ }
+
+ public override void Collect(int doc)
+ {
+ var d = reader.Document(doc);
+ var values = d.GetValues(Syns2Index.F_SYN);
+ foreach (var syn in values.Where(syn => !already.Contains(syn)))
+ {
+ already.Add(syn);
+
+ var tq = new TermQuery(new Term(field, syn));
+ if (boost > 0) // else keep normal 1.0
+ tq.SetBoost(boost);
+
+ tmp.Add(tq, BooleanClause.Occur.SHOULD);
+ }
+ }
+
+ public override void SetNextReader(IndexReader reader, int docBase)
+ {
+ this.reader = reader;
+ }
+
+ public override bool AcceptsDocsOutOfOrder()
+ {
+ return true;
+ }
+
+ }
+
+ /// <summary>
+ /// From project WordNet.Net.Syns2Index
+ /// </summary>
+ public class Syns2Index
+ {
+ /// <summary> </summary>
+ public const String F_SYN = "syn";
+
+ /// <summary> </summary>
+ public const String F_WORD = "word";
+ }
- /// <summary>
- /// From project WordNet.Net.Syns2Index
- /// </summary>
- public class Syns2Index
- {
- /// <summary> </summary>
- public const System.String F_SYN = "syn";
-
- /// <summary> </summary>
- public const System.String F_WORD = "word";
}
+
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/src/contrib/WordNet/Syns2Index/Syns2Index.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/contrib/WordNet/Syns2Index/Syns2Index.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/contrib/WordNet/Syns2Index/Syns2Index.cs (original)
+++ incubator/lucene.net/trunk/src/contrib/WordNet/Syns2Index/Syns2Index.cs Tue Feb 28 22:43:08 2012
@@ -16,8 +16,12 @@
*/
using System;
-
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Lucene.Net.Store;
using Analyzer = Lucene.Net.Analysis.Analyzer;
+using Directory = System.IO.Directory;
using StandardAnalyzer = Lucene.Net.Analysis.Standard.StandardAnalyzer;
using Document = Lucene.Net.Documents.Document;
using Field = Lucene.Net.Documents.Field;
@@ -44,18 +48,12 @@ namespace WorldNet.Net
/// While the WordNet file distinguishes groups of synonyms with
/// related meanings we don't do that here.
/// </p>
- ///
/// This can take 4 minutes to execute and build an index on a "fast" system and the index takes up almost 3 MB.
- ///
/// </summary>
- /// <author> Dave Spencer, dave@searchmorph.com
- /// </author>
- /// <seealso cref="href="http://www.cogsci.princeton.edu/~wn/">WordNet home page</a>">
- /// </seealso>
- /// <seealso cref="href="http://www.cogsci.princeton.edu/~wn/man/prologdb.5WN.html">prologdb man page</a>">
- /// </seealso>
- /// <seealso cref="href="http://www.hostmon.com/rfc/advanced.jsp">sample site that uses it</a>">
- /// </seealso>
+ ///
+ /// <seealso cref="http://www.cogsci.princeton.edu/~wn/"></seealso>
+ /// <seealso cref="http://www.cogsci.princeton.edu/~wn/man/prologdb.5WN.html"></seealso>
+ /// <seealso cref="http://www.hostmon.com/rfc/advanced.jsp"> </seealso>
public class Syns2Index
{
/// <summary> </summary>
@@ -71,15 +69,17 @@ namespace WorldNet.Net
public const System.String F_WORD = "word";
/// <summary> </summary>
- private static readonly Analyzer ana = new StandardAnalyzer();
+ private static readonly Analyzer ana = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_CURRENT);
- /// <summary> Takes arg of prolog file name and index directory.</summary>
+ /// <summary>
+ /// Takes arg of prolog file name and index directory.
+ /// </summary>
[STAThread]
public static void Main(System.String[] args)
{
// get command line arguments
- System.String prologFilename = null; // name of file "wn_s.pl"
- System.String indexDir = null;
+ String prologFilename = null; // name of file "wn_s.pl"
+ String indexDir = null;
if (args.Length == 2)
{
prologFilename = args[0];
@@ -88,45 +88,44 @@ namespace WorldNet.Net
else
{
Usage();
- System.Environment.Exit(1);
+ Environment.Exit(1);
}
// ensure that the prolog file is readable
- if (!(new System.IO.FileInfo(prologFilename)).Exists)
+ if (!(new FileInfo(prologFilename)).Exists)
{
err.WriteLine("Error: cannot read Prolog file: " + prologFilename);
- System.Environment.Exit(1);
+ Environment.Exit(1);
}
// exit if the target index directory already exists
- if (System.IO.Directory.Exists((new System.IO.FileInfo(indexDir)).FullName))
+ if (Directory.Exists((new FileInfo(indexDir)).FullName))
{
err.WriteLine("Error: index directory already exists: " + indexDir);
err.WriteLine("Please specify a name of a non-existent directory");
- System.Environment.Exit(1);
+ Environment.Exit(1);
}
o.WriteLine("Opening Prolog file " + prologFilename);
- System.IO.FileStream fis = new System.IO.FileStream(prologFilename, System.IO.FileMode.Open, System.IO.FileAccess.Read);
- System.IO.StreamReader br = new System.IO.StreamReader(new System.IO.StreamReader(fis, System.Text.Encoding.Default).BaseStream, new System.IO.StreamReader(fis, System.Text.Encoding.Default).CurrentEncoding);
- System.String line;
+ var fis = new FileStream(prologFilename, FileMode.Open, FileAccess.Read);
+ var br = new StreamReader(new StreamReader(fis, System.Text.Encoding.Default).BaseStream, new StreamReader(fis, System.Text.Encoding.Default).CurrentEncoding);
+ String line;
// maps a word to all the "groups" it's in
System.Collections.IDictionary word2Nums = new System.Collections.SortedList();
// maps a group to all the words in it
System.Collections.IDictionary num2Words = new System.Collections.SortedList();
// number of rejected words
- int ndecent = 0;
+ var ndecent = 0;
// status output
- int mod = 1;
- int row = 1;
+ var mod = 1;
+ var row = 1;
// parse prolog file
o.WriteLine("[1/2] Parsing " + prologFilename);
while ((line = br.ReadLine()) != null)
{
// occasional progress
- if ((++row) % mod == 0)
- // periodically print out line we read in
+ if ((++row) % mod == 0) // periodically print out line we read in
{
mod *= 2;
o.WriteLine("\t" + row + " " + line + " " + word2Nums.Count + " " + num2Words.Count + " ndecent=" + ndecent);
@@ -136,17 +135,17 @@ namespace WorldNet.Net
if (!line.StartsWith("s("))
{
err.WriteLine("OUCH: " + line);
- System.Environment.Exit(1);
+ Environment.Exit(1);
}
// parse line
line = line.Substring(2);
- int comma = line.IndexOf((System.Char) ',');
- System.String num = line.Substring(0, (comma) - (0));
- int q1 = line.IndexOf((System.Char) '\'');
+ var comma = line.IndexOf(',');
+ var num = line.Substring(0, comma);
+ var q1 = line.IndexOf('\'');
line = line.Substring(q1 + 1);
- int q2 = line.IndexOf((System.Char) '\'');
- System.String word = line.Substring(0, (q2) - (0)).ToLower();
+ var q2 = line.IndexOf('\'');
+ var word = line.Substring(0, q2).ToLower().Replace("''", "'");
// make sure is a normal word
if (!IsDecent(word))
@@ -157,11 +156,10 @@ namespace WorldNet.Net
// 1/2: word2Nums map
// append to entry or add new one
- System.Collections.IList lis = (System.Collections.IList) word2Nums[word];
+ var lis = (System.Collections.IList) word2Nums[word];
if (lis == null)
{
- lis = new System.Collections.ArrayList();
- lis.Add(num);
+ lis = new List<String> {num};
word2Nums[word] = lis;
}
else
@@ -171,8 +169,7 @@ namespace WorldNet.Net
lis = (System.Collections.IList) num2Words[num];
if (lis == null)
{
- lis = new System.Collections.ArrayList();
- lis.Add(word);
+ lis = new List<String> { word };
num2Words[num] = lis;
}
else
@@ -188,20 +185,18 @@ namespace WorldNet.Net
Index(indexDir, word2Nums, num2Words);
}
- /// <summary> Checks to see if a word contains only alphabetic characters by
+ /// <summary>
+ /// Checks to see if a word contains only alphabetic characters by
/// checking it one character at a time.
- ///
/// </summary>
- /// <param name="s">string to check
- /// </param>
- /// <returns> <c>true</c> if the string is decent
- /// </returns>
- private static bool IsDecent(System.String s)
+ /// <param name="s">string to check </param>
+ /// <returns> <c>true</c> if the string is decent</returns>
+ private static bool IsDecent(String s)
{
- int len = s.Length;
- for (int i = 0; i < len; i++)
+ var len = s.Length;
+ for (var i = 0; i < len; i++)
{
- if (!System.Char.IsLetter(s[i]))
+ if (!Char.IsLetter(s[i]))
{
return false;
}
@@ -209,75 +204,73 @@ namespace WorldNet.Net
return true;
}
- /// <summary> Forms a Lucene index based on the 2 maps.
- ///
+ /// <summary>
+ /// Forms a Lucene index based on the 2 maps.
/// </summary>
- /// <param name="indexDir">the direcotry where the index should be created
- /// </param>
- /// <param name="">word2Nums
- /// </param>
- /// <param name="">num2Words
- /// </param>
- private static void Index(System.String indexDir, System.Collections.IDictionary word2Nums, System.Collections.IDictionary num2Words)
+ /// <param name="indexDir">the direcotry where the index should be created</param>
+ /// <param name="word2Nums">word2Nums</param>
+ /// <param name="num2Words">num2Words</param>
+ private static void Index(String indexDir, System.Collections.IDictionary word2Nums, System.Collections.IDictionary num2Words)
{
- int row = 0;
- int mod = 1;
+ var row = 0;
+ var mod = 1;
- // override the specific index if it already exists
- IndexWriter writer = new IndexWriter(indexDir, ana, true);
- writer.SetUseCompoundFile(true); // why?
- // blindly up these parameters for speed
- writer.SetMergeFactor(writer.GetMergeFactor() * 2);
- writer.SetMaxBufferedDocs(writer.GetMaxBufferedDocs() * 2);
- System.Collections.IEnumerator i1 = word2Nums.Keys.GetEnumerator();
- while (i1.MoveNext())
- // for each word
+ using (var dir = FSDirectory.Open(new DirectoryInfo(indexDir)))
{
- System.String g = (System.String) i1.Current;
- Document doc = new Document();
-
- int n = Index(word2Nums, num2Words, g, doc);
- if (n > 0)
+ var writer = new IndexWriter(dir, ana, true, IndexWriter.MaxFieldLength.LIMITED);
+ writer.SetUseCompoundFile(true); // why?
+
+ var i1 = word2Nums.Keys.GetEnumerator();
+ while (i1.MoveNext())
{
- doc.Add(new Field(F_WORD, g, Field.Store.YES, Field.Index.UN_TOKENIZED));
- if ((++row % mod) == 0)
+ var g = (String)i1.Current;
+ var doc = new Document();
+
+ var n = Index(word2Nums, num2Words, g, doc);
+ if (n > 0)
{
- o.WriteLine("\trow=" + row + "/" + word2Nums.Count + " doc= " + doc);
- mod *= 2;
+ doc.Add(new Field(F_WORD, g, Field.Store.YES, Field.Index.NOT_ANALYZED));
+ if ((++row % mod) == 0)
+ {
+ o.WriteLine("\trow=" + row + "/" + word2Nums.Count + " doc= " + doc);
+ mod *= 2;
+ }
+ writer.AddDocument(doc);
}
- writer.AddDocument(doc);
- } // else degenerate
+ }
+ o.WriteLine("Optimizing..");
+ writer.Optimize();
+ writer.Close();
}
- o.WriteLine("Optimizing..");
- writer.Optimize();
- writer.Close();
+
}
- /// <summary> Given the 2 maps fills a document for 1 word.</summary>
+ /// <summary>
+ /// Given the 2 maps fills a document for 1 word.
+ /// </summary>
private static int Index(System.Collections.IDictionary word2Nums, System.Collections.IDictionary num2Words, System.String g, Document doc)
{
- System.Collections.IList keys = (System.Collections.IList) word2Nums[g]; // get list of key#'s
- System.Collections.IEnumerator i2 = keys.GetEnumerator();
+ var keys = (System.Collections.IList) word2Nums[g]; // get list of key#'s
+ var i2 = keys.GetEnumerator();
- System.Collections.SortedList already = new System.Collections.SortedList(); // keep them sorted
+ var already = new System.Collections.SortedList(); // keep them sorted
// pass 1: fill up 'already' with all words
while (i2.MoveNext()) // for each key#
{
- foreach (object item in (System.Collections.IList) num2Words[i2.Current]) // get list of words
+ foreach (var item in
+ ((System.Collections.IList) num2Words[i2.Current]).Cast<object>().Where(item => already.Contains(item) == false))
{
- if (already.Contains(item) == false)
- {
- already.Add(item, item);
- }
+ already.Add(item, item);
}
}
- int num = 0;
+
+ var num = 0;
already.Remove(g); // of course a word is it's own syn
- System.Collections.IDictionaryEnumerator it = already.GetEnumerator();
+ var it = already.GetEnumerator();
while (it.MoveNext())
{
- System.String cur = (System.String) it.Key;
+ var cur = (String) it.Key;
// don't store things like 'pit bull' -> 'american pit bull'
if (!IsDecent(cur))
{
@@ -295,16 +288,5 @@ namespace WorldNet.Net
o.WriteLine("\n\n" + typeof(Syns2Index) + " <prolog file> <index dir>\n\n");
}
- static Syns2Index()
- {
- System.IO.StreamWriter temp_writer;
- temp_writer = new System.IO.StreamWriter(System.Console.OpenStandardOutput(), System.Console.Out.Encoding);
- temp_writer.AutoFlush = true;
- o = temp_writer;
- System.IO.StreamWriter temp_writer2;
- temp_writer2 = new System.IO.StreamWriter(System.Console.OpenStandardError(), System.Console.Error.Encoding);
- temp_writer2.AutoFlush = true;
- err = temp_writer2;
- }
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/src/core/Analysis/ASCIIFoldingFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/ASCIIFoldingFilter.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/ASCIIFoldingFilter.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/ASCIIFoldingFilter.cs Tue Feb 28 22:43:08 2012
@@ -56,11 +56,11 @@ namespace Lucene.Net.Analysis
/// accents from Latin1 characters. For example, 'À' will be replaced by
/// 'a'.
/// </summary>
- public sealed class ASCIIFoldingFilter:TokenFilter
+ public sealed class ASCIIFoldingFilter : TokenFilter
{
public ASCIIFoldingFilter(TokenStream input):base(input)
{
- termAtt = (TermAttribute) AddAttribute(typeof(TermAttribute));
+ termAtt = AddAttribute<TermAttribute>();
}
private char[] output = new char[512];
Modified: incubator/lucene.net/trunk/src/core/Analysis/Analyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/Analyzer.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/Analyzer.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/Analyzer.cs Tue Feb 28 22:43:08 2012
@@ -16,10 +16,9 @@
*/
using System;
-
+using Lucene.Net.Store;
+using Lucene.Net.Util;
using Fieldable = Lucene.Net.Documents.Fieldable;
-using AlreadyClosedException = Lucene.Net.Store.AlreadyClosedException;
-using CloseableThreadLocal = Lucene.Net.Util.CloseableThreadLocal;
namespace Lucene.Net.Analysis
{
@@ -30,7 +29,7 @@ namespace Lucene.Net.Analysis
/// characters from the Reader into raw Tokens. One or more TokenFilters may
/// then be applied to the output of the Tokenizer.
/// </summary>
- public abstract class Analyzer
+ public abstract class Analyzer : IDisposable
{
/// <summary>Creates a TokenStream which tokenizes all the text in the provided
/// Reader. Must be able to handle null field name for
@@ -50,84 +49,81 @@ namespace Lucene.Net.Analysis
return TokenStream(fieldName, reader);
}
- private CloseableThreadLocal tokenStreams = new CloseableThreadLocal();
-
+ private CloseableThreadLocal<Object> tokenStreams = new CloseableThreadLocal<Object>();
+ private bool isDisposed;
+
+ /// <summary>Used by Analyzers that implement reusableTokenStream
+ /// to save a TokenStream for later re-use by the same
+ /// thread.
+ /// </summary>
+ protected internal virtual object PreviousTokenStream
+ {
+ get
+ {
+ if (tokenStreams == null)
+ {
+ throw new AlreadyClosedException("this Analyzer is closed");
+ }
+ return tokenStreams.Get();
+ }
+
+ set
+ {
+ if (tokenStreams == null)
+ {
+ throw new AlreadyClosedException("this Analyzer is closed");
+ }
+ tokenStreams.Set(value);
+ }
+ }
+
/// <summary>Used by Analyzers that implement reusableTokenStream
/// to retrieve previously saved TokenStreams for re-use
/// by the same thread.
- /// </summary>
+ /// </summary>
+ [Obsolete("Use PreviousTokenStream property instead")]
protected internal virtual System.Object GetPreviousTokenStream()
{
- try
- {
- return tokenStreams.Get();
- }
- catch (System.NullReferenceException npe)
- {
- if (tokenStreams == null)
- {
- throw new AlreadyClosedException("this Analyzer is closed");
- }
- else
- {
- throw npe;
- }
- }
+ return PreviousTokenStream;
}
/// <summary>Used by Analyzers that implement reusableTokenStream
/// to save a TokenStream for later re-use by the same
/// thread.
/// </summary>
+ [Obsolete("Use PreviousTokenStream property instead")]
protected internal virtual void SetPreviousTokenStream(System.Object obj)
{
- try
- {
- tokenStreams.Set(obj);
- }
- catch (System.NullReferenceException npe)
- {
- if (tokenStreams == null)
- {
- throw new AlreadyClosedException("this Analyzer is closed");
- }
- else
- {
- throw npe;
- }
- }
+ PreviousTokenStream = obj;
}
- protected internal bool overridesTokenStreamMethod;
+ [Obsolete()]
+ protected internal bool overridesTokenStreamMethod = false;
/// <deprecated> This is only present to preserve
/// back-compat of classes that subclass a core analyzer
/// and override tokenStream but not reusableTokenStream
/// </deprecated>
+ /// <summary>
+ /// Java uses Class<? extends Analyer> to contrain <paramref="baseClass"/> to
+ /// only Types that inherit from Analyzer. C# does not have a generic type class,
+ /// ie Type<t>. The method signature stays the same, and an exception may
+ /// still be thrown, if the method doesn't exist.
+ /// </summary>
[Obsolete("This is only present to preserve back-compat of classes that subclass a core analyzer and override tokenStream but not reusableTokenStream ")]
- protected internal virtual void SetOverridesTokenStreamMethod(System.Type baseClass)
+ protected internal virtual void SetOverridesTokenStreamMethod<TClass>()
+ where TClass : Analyzer
{
-
- System.Type[] params_Renamed = new System.Type[2];
- params_Renamed[0] = typeof(System.String);
- params_Renamed[1] = typeof(System.IO.TextReader);
-
- try
- {
- System.Reflection.MethodInfo m = this.GetType().GetMethod("TokenStream", (params_Renamed == null)?new System.Type[0]:(System.Type[]) params_Renamed);
- if (m != null)
- {
- overridesTokenStreamMethod = m.DeclaringType != baseClass;
- }
- else
- {
- overridesTokenStreamMethod = false;
- }
- }
- catch (System.MethodAccessException nsme)
- {
- overridesTokenStreamMethod = false;
- }
+ try
+ {
+ System.Reflection.MethodInfo m = this.GetType().GetMethod("TokenStream", new[] { typeof(string), typeof(System.IO.TextReader) });
+ overridesTokenStreamMethod = m.DeclaringType != typeof(TClass);
+ }
+ catch (System.MethodAccessException nsme)
+ {
+ // can't happen, as baseClass is subclass of Analyzer
+ overridesTokenStreamMethod = false;
+ }
}
@@ -171,10 +167,29 @@ namespace Lucene.Net.Analysis
}
/// <summary>Frees persistent resources used by this Analyzer </summary>
- public virtual void Close()
+ public void Close()
{
- tokenStreams.Close();
- tokenStreams = null;
+ Dispose();
}
+
+ public virtual void Dispose()
+ {
+ Dispose(true);
+ }
+
+ protected virtual void Dispose(bool disposing)
+ {
+ if (isDisposed) return;
+
+ if (disposing)
+ {
+ if (tokenStreams != null)
+ {
+ tokenStreams.Close();
+ tokenStreams = null;
+ }
+ }
+ isDisposed = true;
+ }
}
}
\ No newline at end of file
Modified: incubator/lucene.net/trunk/src/core/Analysis/BaseCharFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/BaseCharFilter.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/BaseCharFilter.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/BaseCharFilter.cs Tue Feb 28 22:43:08 2012
@@ -16,6 +16,7 @@
*/
using System;
+using Lucene.Net.Support;
using Lucene.Net.Util;
namespace Lucene.Net.Analysis
@@ -34,13 +35,13 @@ namespace Lucene.Net.Analysis
private int[] diffs;
private int size = 0;
- public BaseCharFilter(CharStream @in) : base(@in)
+ protected BaseCharFilter(CharStream @in) : base(@in)
{
}
/** Retrieve the corrected offset. */
//@Override
- public override int Correct(int currentOff)
+ protected internal override int Correct(int currentOff)
{
if (offsets == null || currentOff < offsets[0])
{
@@ -56,7 +57,7 @@ namespace Lucene.Net.Analysis
while (hi >= lo)
{
- mid = SupportClass.Number.URShift(lo + hi, 1);
+ mid = Number.URShift(lo + hi, 1);
if (currentOff < offsets[mid])
hi = mid - 1;
else if (currentOff > offsets[mid])
@@ -71,10 +72,18 @@ namespace Lucene.Net.Analysis
return currentOff + diffs[mid];
}
+ protected int LastCumulativeDiff
+ {
+ get
+ {
+ return offsets == null ? 0 : diffs[size - 1];
+ }
+ }
+
+ [Obsolete("Use LastCumulativeDiff property instead")]
protected int GetLastCumulativeDiff()
{
- return offsets == null ?
- 0 : diffs[size - 1];
+ return LastCumulativeDiff;
}
protected void AddOffCorrectMap(int off, int cumulativeDiff)
Modified: incubator/lucene.net/trunk/src/core/Analysis/CachingTokenFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/trunk/src/core/Analysis/CachingTokenFilter.cs?rev=1294875&r1=1294874&r2=1294875&view=diff
==============================================================================
--- incubator/lucene.net/trunk/src/core/Analysis/CachingTokenFilter.cs (original)
+++ incubator/lucene.net/trunk/src/core/Analysis/CachingTokenFilter.cs Tue Feb 28 22:43:08 2012
@@ -16,8 +16,7 @@
*/
using System;
-
-using AttributeSource = Lucene.Net.Util.AttributeSource;
+using Lucene.Net.Util;
namespace Lucene.Net.Analysis
{
@@ -30,40 +29,22 @@ namespace Lucene.Net.Analysis
/// <see cref="TokenStream.Reset()" />, which repositions the
/// stream to the first Token.
/// </summary>
- public class CachingTokenFilter:TokenFilter
+ public sealed class CachingTokenFilter : TokenFilter
{
- private System.Collections.IList cache = null;
- private System.Collections.IEnumerator iterator = null;
+ private System.Collections.Generic.LinkedList<AttributeSource.State> cache = null;
+ private System.Collections.Generic.IEnumerator<State> iterator = null;
private AttributeSource.State finalState;
public CachingTokenFilter(TokenStream input):base(input)
{
}
-
- /// <deprecated> Will be removed in Lucene 3.0. This method is final, as it should
- /// not be overridden. Delegates to the backwards compatibility layer.
- /// </deprecated>
- [Obsolete("Will be removed in Lucene 3.0. This method is final, as it should not be overridden. Delegates to the backwards compatibility layer. ")]
- public override Token Next(Token reusableToken)
- {
- return base.Next(reusableToken);
- }
-
- /// <deprecated> Will be removed in Lucene 3.0. This method is final, as it should
- /// not be overridden. Delegates to the backwards compatibility layer.
- /// </deprecated>
- [Obsolete("Will be removed in Lucene 3.0. This method is final, as it should not be overridden. Delegates to the backwards compatibility layer. ")]
- public override Token Next()
- {
- return base.Next();
- }
-
+
public override bool IncrementToken()
{
if (cache == null)
{
// fill cache lazily
- cache = new System.Collections.ArrayList();
+ cache = new System.Collections.Generic.LinkedList<State>();
FillCache();
iterator = cache.GetEnumerator();
}
@@ -74,7 +55,7 @@ namespace Lucene.Net.Analysis
return false;
}
// Since the TokenFilter can be reset, the tokens need to be preserved as immutable.
- RestoreState((AttributeSource.State) iterator.Current);
+ RestoreState(iterator.Current);
return true;
}
@@ -98,7 +79,7 @@ namespace Lucene.Net.Analysis
{
while (input.IncrementToken())
{
- cache.Add(CaptureState());
+ cache.AddLast(CaptureState());
}
// capture final state
input.End();