You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by cc...@apache.org on 2011/11/23 09:38:52 UTC
[Lucene.Net] svn commit: r1205326 - in
/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk: build/vs2010/test/
src/contrib/Snowball/Lucene.Net/Analysis/Snowball/
src/contrib/Snowball/SF/Snowball/ src/contrib/Snowball/SF/Snowball/Ext/
test/contrib/SimpleFacetedSea...
Author: ccurrens
Date: Wed Nov 23 08:38:49 2011
New Revision: 1205326
URL: http://svn.apache.org/viewvc?rev=1205326&view=rev
Log:
Ported Contrib.Snowball
Modified:
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Regex.Test.sln
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Snowball.Test.sln
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballAnalyzer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballFilter.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DanishStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DutchStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/EnglishStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FinnishStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FrenchStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/German2Stemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/GermanStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/HungarianStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/ItalianStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/KpStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/LovinsStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/NorwegianStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PorterStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PortugueseStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RomanianStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RussianStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SpanishStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SwedishStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/TurkishStemmer.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/SnowballProgram.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/SimpleFacetedSearch/TestSimpleFacetedSearch.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Analysis/Snowball/TestSnowball.cs
incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Contrib.Snowball.Test.csproj
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Regex.Test.sln
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Regex.Test.sln?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Regex.Test.sln (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Regex.Test.sln Wed Nov 23 08:38:49 2011
@@ -1,6 +1,26 @@

Microsoft Visual Studio Solution File, Format Version 11.00
-# Visual Studio 2010
+# Visual C# Express 2010
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net", "..\..\..\src\core\Lucene.Net.csproj", "{5D4AD9BE-1FFB-41AB-9943-25737971BF57}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Test", "..\..\..\test\core\Lucene.Net.Test.csproj", "{AAF68BCF-F781-45FC-98B3-2B9CEE411E01}"
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Snowball.Test.sln
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Snowball.Test.sln?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Snowball.Test.sln (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/build/vs2010/test/Contrib.Snowball.Test.sln Wed Nov 23 08:38:49 2011
@@ -1,32 +1,14 @@

Microsoft Visual Studio Solution File, Format Version 11.00
-# Visual C# Express 2010
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
+# Visual Studio 2010
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net", "..\..\..\src\core\Lucene.Net.csproj", "{5D4AD9BE-1FFB-41AB-9943-25737971BF57}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Contrib.Snowball", "..\..\..\src\contrib\Snowball\Contrib.Snowball.csproj", "{8F9D7A92-F122-413E-9D8D-027E4ECD327C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Contrib.Snowball.Test", "..\..\..\test\contrib\Snowball\Contrib.Snowball.Test.csproj", "{992216FA-372D-4412-8D7C-E252AE042F48}"
EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Lucene.Net.Test", "..\..\..\test\core\Lucene.Net.Test.csproj", "{AAF68BCF-F781-45FC-98B3-2B9CEE411E01}"
+EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -45,6 +27,10 @@ Global
{992216FA-372D-4412-8D7C-E252AE042F48}.Debug|Any CPU.Build.0 = Debug|Any CPU
{992216FA-372D-4412-8D7C-E252AE042F48}.Release|Any CPU.ActiveCfg = Release|Any CPU
{992216FA-372D-4412-8D7C-E252AE042F48}.Release|Any CPU.Build.0 = Release|Any CPU
+ {AAF68BCF-F781-45FC-98B3-2B9CEE411E01}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {AAF68BCF-F781-45FC-98B3-2B9CEE411E01}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {AAF68BCF-F781-45FC-98B3-2B9CEE411E01}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {AAF68BCF-F781-45FC-98B3-2B9CEE411E01}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballAnalyzer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballAnalyzer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballAnalyzer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballAnalyzer.cs Wed Nov 23 08:38:49 2011
@@ -16,48 +16,110 @@
*/
using System;
+using System.Collections.Generic;
+using System.IO;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Standard;
using SF.Snowball.Ext;
+using Version = Lucene.Net.Util.Version;
+
namespace Lucene.Net.Analysis.Snowball
{
-
- /// <summary>Filters <see cref="StandardTokenizer"/> with <see cref="StandardFilter"/>, {@link
- /// LowerCaseFilter}, <see cref="StopFilter"/> and <see cref="SnowballFilter"/>.
- ///
- /// Available stemmers are listed in <see cref="SF.Snowball.Ext"/>. The name of a
- /// stemmer is the part of the class name before "Stemmer", e.g., the stemmer in
- /// <see cref="EnglishStemmer"/> is named "English".
- /// </summary>
- public class SnowballAnalyzer : Analyzer
- {
- private System.String name;
- private System.Collections.Hashtable stopSet;
-
- /// <summary>Builds the named analyzer with no stop words. </summary>
- public SnowballAnalyzer(System.String name)
- {
- this.name = name;
- }
-
- /// <summary>Builds the named analyzer with the given stop words. </summary>
- public SnowballAnalyzer(System.String name, System.String[] stopWords) : this(name)
- {
- stopSet = StopFilter.MakeStopSet(stopWords);
- }
-
- /// <summary>Constructs a <see cref="StandardTokenizer"/> filtered by a {@link
- /// StandardFilter}, a <see cref="LowerCaseFilter"/> and a <see cref="StopFilter"/>.
- /// </summary>
+
+ /// <summary>Filters <see cref="StandardTokenizer"/> with <see cref="StandardFilter"/>, {@link
+ /// LowerCaseFilter}, <see cref="StopFilter"/> and <see cref="SnowballFilter"/>.
+ ///
+ /// Available stemmers are listed in <see cref="SF.Snowball.Ext"/>. The name of a
+ /// stemmer is the part of the class name before "Stemmer", e.g., the stemmer in
+ /// <see cref="EnglishStemmer"/> is named "English".
+ ///
+ /// <p><b>NOTE:</b> This class uses the same <see cref="Version"/>
+ /// dependent settings as <see cref="StandardAnalyzer"/></p>
+ /// </summary>
+ public class SnowballAnalyzer : Analyzer
+ {
+ private System.String name;
+ private ISet<string> stopSet;
+ private readonly Version matchVersion;
+
+ /// <summary>Builds the named analyzer with no stop words. </summary>
+ public SnowballAnalyzer(Version matchVersion, System.String name)
+ {
+ this.name = name;
+ SetOverridesTokenStreamMethod(typeof(SnowballAnalyzer));
+ this.matchVersion = matchVersion;
+ }
+
+ /// <summary>Builds the named analyzer with the given stop words. </summary>
+ [Obsolete("Use SnowballAnalyzer(Version, string, ISet) instead.")]
+ public SnowballAnalyzer(Version matchVersion, System.String name, System.String[] stopWords)
+ : this(matchVersion, name)
+ {
+ stopSet = StopFilter.MakeStopSet(stopWords);
+ }
+
+ /// <summary>
+ /// Builds the named analyzer with the given stop words.
+ /// </summary>
+ public SnowballAnalyzer(Version matchVersion, string name, ISet<string> stopWords)
+ : this(matchVersion, name)
+ {
+ stopSet = CharArraySet.UnmodifiableSet(CharArraySet.Copy(stopWords));
+ }
+
+ /// <summary>Constructs a <see cref="StandardTokenizer"/> filtered by a {@link
+ /// StandardFilter}, a <see cref="LowerCaseFilter"/> and a <see cref="StopFilter"/>.
+ /// </summary>
public override TokenStream TokenStream(System.String fieldName, System.IO.TextReader reader)
- {
- TokenStream result = new StandardTokenizer(reader);
- result = new StandardFilter(result);
- result = new LowerCaseFilter(result);
- if (stopSet != null)
- result = new StopFilter(result, stopSet);
- result = new SnowballFilter(result, name);
- return result;
- }
- }
+ {
+ TokenStream result = new StandardTokenizer(matchVersion, reader);
+ result = new StandardFilter(result);
+ result = new LowerCaseFilter(result);
+ if (stopSet != null)
+ result = new StopFilter(StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion),
+ result, stopSet);
+ result = new SnowballFilter(result, name);
+ return result;
+ }
+
+ private class SavedStreams
+ {
+ internal Tokenizer source;
+ internal TokenStream result;
+ };
+
+ /** Returns a (possibly reused) {@link StandardTokenizer} filtered by a
+ * {@link StandardFilter}, a {@link LowerCaseFilter},
+ * a {@link StopFilter}, and a {@link SnowballFilter} */
+
+ public override TokenStream ReusableTokenStream(String fieldName, TextReader reader)
+ {
+ if (overridesTokenStreamMethod)
+ {
+ // LUCENE-1678: force fallback to tokenStream() if we
+ // have been subclassed and that subclass overrides
+ // tokenStream but not reusableTokenStream
+ return TokenStream(fieldName, reader);
+ }
+
+ SavedStreams streams = (SavedStreams)GetPreviousTokenStream();
+ if (streams == null)
+ {
+ streams = new SavedStreams();
+ streams.source = new StandardTokenizer(matchVersion, reader);
+ streams.result = new StandardFilter(streams.source);
+ streams.result = new LowerCaseFilter(streams.result);
+ if (stopSet != null)
+ streams.result = new StopFilter(StopFilter.GetEnablePositionIncrementsVersionDefault(matchVersion),
+ streams.result, stopSet);
+ streams.result = new SnowballFilter(streams.result, name);
+ SetPreviousTokenStream(streams);
+ }
+ else
+ {
+ streams.source.Reset(reader);
+ }
+ return streams.result;
+ }
+ }
}
\ No newline at end of file
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballFilter.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballFilter.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballFilter.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/Lucene.Net/Analysis/Snowball/SnowballFilter.cs Wed Nov 23 08:38:49 2011
@@ -16,6 +16,7 @@
*/
using System;
+using Lucene.Net.Analysis.Tokenattributes;
using Token = Lucene.Net.Analysis.Token;
using TokenFilter = Lucene.Net.Analysis.TokenFilter;
using TokenStream = Lucene.Net.Analysis.TokenStream;
@@ -32,54 +33,60 @@ namespace Lucene.Net.Analysis.Snowball
/// <see cref="EnglishStemmer"/> is named "English".
/// </summary>
- public class SnowballFilter : TokenFilter
+ public sealed class SnowballFilter : TokenFilter
{
private static readonly System.Object[] EMPTY_ARGS = new System.Object[0];
private SnowballProgram stemmer;
- private System.Reflection.MethodInfo stemMethod;
-
+ private TermAttribute termAtt;
+ //private System.Reflection.MethodInfo stemMethod;
+
+ public SnowballFilter(TokenStream input, SnowballProgram stemmer)
+ : base(input)
+ {
+ this.stemmer = stemmer;
+ termAtt = AddAttribute<TermAttribute>();
+ }
+
/// <summary>Construct the named stemming filter.
///
/// </summary>
- /// <param name="in_Renamed">the input tokens to stem
+ /// <param name="input">the input tokens to stem
/// </param>
/// <param name="name">the name of a stemmer
/// </param>
- public SnowballFilter(TokenStream in_Renamed, System.String name) : base(in_Renamed)
+ public SnowballFilter(TokenStream input, System.String name) : base(input)
{
try
{
System.Type stemClass = System.Type.GetType("SF.Snowball.Ext." + name + "Stemmer");
stemmer = (SnowballProgram) System.Activator.CreateInstance(stemClass);
- // why doesn't the SnowballProgram class have an (abstract?) stem method?
- stemMethod = stemClass.GetMethod("Stem", (new System.Type[0] == null) ? new System.Type[0] : (System.Type[]) new System.Type[0]);
}
catch (System.Exception e)
{
throw new System.SystemException(e.ToString());
}
+ termAtt = AddAttribute<TermAttribute>();
}
/// <summary>Returns the next input Token, after being stemmed </summary>
- public override Token Next()
+ public sealed override bool IncrementToken()
{
- Token token = input.Next();
- if (token == null)
- return null;
- stemmer.SetCurrent(token.TermText());
- try
- {
- stemMethod.Invoke(stemmer, (System.Object[]) EMPTY_ARGS);
- }
- catch (System.Exception e)
- {
- throw new System.SystemException(e.ToString());
- }
-
- Token newToken = new Token(stemmer.GetCurrent(), token.StartOffset(), token.EndOffset(), token.Type());
- newToken.SetPositionIncrement(token.GetPositionIncrement());
- return newToken;
+ if (input.IncrementToken())
+ {
+ String originalTerm = termAtt.Term();
+ stemmer.SetCurrent(originalTerm);
+ stemmer.Stem();
+ String finalTerm = stemmer.GetCurrent();
+ // Don't bother updating, if it is unchanged.
+ if (!originalTerm.Equals(finalTerm))
+ termAtt.SetTermBuffer(finalTerm);
+ return true;
+ }
+ else
+ {
+ return false;
+ }
}
}
}
\ No newline at end of file
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DanishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DanishStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DanishStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DanishStemmer.cs Wed Nov 23 08:38:49 2011
@@ -31,7 +31,7 @@ namespace SF.Snowball.Ext
}
private void InitBlock()
{
- a_0 = new Among[]{new Among("hed", - 1, 1, "", this), new Among("ethed", 0, 1, "", this), new Among("ered", - 1, 1, "", this), new Among("e", - 1, 1, "", this), new Among("erede", 3, 1, "", this), new Among("ende", 3, 1, "", this), new Among("erende", 5, 1, "", this), new Among("ene", 3, 1, "", this), new Among("erne", 3, 1, "", this), new Among("ere", 3, 1, "", this), new Among("en", - 1, 1, "", this), new Among("heden", 10, 1, "", this), new Among("eren", 10, 1, "", this), new Among("er", - 1, 1, "", this), new Among("heder", 13, 1, "", this), new Among("erer", 13, 1, "", this), new Among("s", - 1, 2, "", this), new Among("heds", 16, 1, "", this), new Among("es", 16, 1, "", this), new Among("endes", 18, 1, "", this), new Among("erendes", 19, 1, "", this), new Among("enes", 18, 1, "", this), new Among("ernes", 18, 1, "", this), new Among("eres", 18, 1, "", this), new Among("ens", 16, 1, "", this), new Among("hedens", 24, 1, "", this), new Among("erens", 24, 1, "", this),
new Among("ers", 16, 1, "", this), new Among("ets", 16, 1, "", this), new Among("erets", 28, 1, "", this), new Among("et", - 1, 1, "", this), new Among("eret", 30, 1, "", this)};
+ a_0 = new Among[] { new Among("hed", -1, 1, "", this), new Among("ethed", 0, 1, "", this), new Among("ered", -1, 1, "", this), new Among("e", -1, 1, "", this), new Among("erede", 3, 1, "", this), new Among("ende", 3, 1, "", this), new Among("erende", 5, 1, "", this), new Among("ene", 3, 1, "", this), new Among("erne", 3, 1, "", this), new Among("ere", 3, 1, "", this), new Among("en", -1, 1, "", this), new Among("heden", 10, 1, "", this), new Among("eren", 10, 1, "", this), new Among("er", -1, 1, "", this), new Among("heder", 13, 1, "", this), new Among("erer", 13, 1, "", this), new Among("s", -1, 2, "", this), new Among("heds", 16, 1, "", this), new Among("es", 16, 1, "", this), new Among("endes", 18, 1, "", this), new Among("erendes", 19, 1, "", this), new Among("enes", 18, 1, "", this), new Among("ernes", 18, 1, "", this), new Among("eres", 18, 1, "", this), new Among("ens", 16, 1, "", this), new Among("hedens", 24, 1, "", this), new Among("erens", 24, 1, "", t
his), new Among("ers", 16, 1, "", this), new Among("ets", 16, 1, "", this), new Among("erets", 28, 1, "", this), new Among("et", -1, 1, "", this), new Among("eret", 30, 1, "", this) };
a_1 = new Among[]{new Among("gd", - 1, - 1, "", this), new Among("dt", - 1, - 1, "", this), new Among("gt", - 1, - 1, "", this), new Among("kt", - 1, - 1, "", this)};
a_2 = new Among[]{new Among("ig", - 1, 1, "", this), new Among("lig", 0, 1, "", this), new Among("elig", 1, 1, "", this), new Among("els", - 1, 1, "", this), new Among("l\u00F8st", - 1, 2, "", this)};
}
@@ -363,7 +363,7 @@ lab1_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DutchStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DutchStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DutchStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/DutchStemmer.cs Wed Nov 23 08:38:49 2011
@@ -946,7 +946,7 @@ lab9_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/EnglishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/EnglishStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/EnglishStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/EnglishStemmer.cs Wed Nov 23 08:38:49 2011
@@ -1244,7 +1244,7 @@ replab0_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FinnishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FinnishStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FinnishStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FinnishStemmer.cs Wed Nov 23 08:38:49 2011
@@ -982,7 +982,7 @@ golab6_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FrenchStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FrenchStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FrenchStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/FrenchStemmer.cs Wed Nov 23 08:38:49 2011
@@ -1490,7 +1490,7 @@ lab16_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/German2Stemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/German2Stemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/German2Stemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/German2Stemmer.cs Wed Nov 23 08:38:49 2011
@@ -811,7 +811,7 @@ lab2_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/GermanStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/GermanStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/GermanStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/GermanStemmer.cs Wed Nov 23 08:38:49 2011
@@ -783,7 +783,7 @@ lab2_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/HungarianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/HungarianStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/HungarianStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/HungarianStemmer.cs Wed Nov 23 08:38:49 2011
@@ -1125,7 +1125,7 @@ namespace SF.Snowball.Ext
return true;
}
- public bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/ItalianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/ItalianStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/ItalianStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/ItalianStemmer.cs Wed Nov 23 08:38:49 2011
@@ -1124,7 +1124,7 @@ lab1_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/KpStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/KpStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/KpStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/KpStemmer.cs Wed Nov 23 08:38:49 2011
@@ -2169,7 +2169,7 @@ lab1_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/LovinsStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/LovinsStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/LovinsStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/LovinsStemmer.cs Wed Nov 23 08:38:49 2011
@@ -1876,7 +1876,7 @@ lab7_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/NorwegianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/NorwegianStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/NorwegianStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/NorwegianStemmer.cs Wed Nov 23 08:38:49 2011
@@ -277,7 +277,7 @@ lab4_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PorterStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PorterStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PorterStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PorterStemmer.cs Wed Nov 23 08:38:49 2011
@@ -653,7 +653,7 @@ lab0_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PortugueseStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PortugueseStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PortugueseStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/PortugueseStemmer.cs Wed Nov 23 08:38:49 2011
@@ -1051,7 +1051,7 @@ namespace SF.Snowball.Ext
return true;
}
- public bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RomanianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RomanianStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RomanianStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RomanianStemmer.cs Wed Nov 23 08:38:49 2011
@@ -976,7 +976,7 @@ namespace SF.Snowball.Ext
return true;
}
- public bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RussianStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RussianStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RussianStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/RussianStemmer.cs Wed Nov 23 08:38:49 2011
@@ -581,7 +581,7 @@ lab3_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SpanishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SpanishStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SpanishStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SpanishStemmer.cs Wed Nov 23 08:38:49 2011
@@ -1104,7 +1104,7 @@ lab5_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
@@ -1147,7 +1147,8 @@ lab1_brk: ;
cursor = limit - v_2;
// do, line 217
- v_3 = limit - cursor;
do
+ v_3 = limit - cursor;
+ do
{
// (, line 217
// or, line 217
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SwedishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SwedishStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SwedishStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/SwedishStemmer.cs Wed Nov 23 08:38:49 2011
@@ -281,7 +281,7 @@ lab4_brk: ;
return true;
}
- public virtual bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/TurkishStemmer.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/TurkishStemmer.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/TurkishStemmer.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/Ext/TurkishStemmer.cs Wed Nov 23 08:38:49 2011
@@ -3052,7 +3052,7 @@ namespace SF.Snowball.Ext
return true;
}
- public bool Stem()
+ public override bool Stem()
{
int v_1;
int v_2;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/SnowballProgram.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/SnowballProgram.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/SnowballProgram.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/src/contrib/Snowball/SF/Snowball/SnowballProgram.cs Wed Nov 23 08:38:49 2011
@@ -15,35 +15,52 @@
* limitations under the License.
*/
using System;
+using System.Text;
+
namespace SF.Snowball
{
-
- public class SnowballProgram
+ /// <summary>
+ /// This is the rev 500 of the snowball SVN trunk,
+ /// but modified:
+ /// made abstract and introduced abstract method stem to avoid expensive reflection in filter class
+ /// </summary>
+ public abstract class SnowballProgram
{
- /// <summary> Get the current string.</summary>
- virtual public System.String GetCurrent()
- {
- return current.ToString();
- }
protected internal SnowballProgram()
{
current = new System.Text.StringBuilder();
SetCurrent("");
}
-
+
+ public abstract bool Stem();
+
/// <summary> Set the current string.</summary>
- public virtual void SetCurrent(System.String value_Renamed)
+ public virtual void SetCurrent(System.String value)
{
//// current.Replace(current.ToString(0, current.Length - 0), value_Renamed, 0, current.Length - 0);
current.Remove(0, current.Length);
- current.Append(value_Renamed);
+ current.Append(value);
cursor = 0;
limit = current.Length;
limit_backward = 0;
bra = cursor;
ket = limit;
}
-
+
+ /// <summary> Get the current string.</summary>
+ virtual public System.String GetCurrent()
+ {
+ string result = current.ToString();
+ // Make a new StringBuffer. If we reuse the old one, and a user of
+ // the library keeps a reference to the buffer returned (for example,
+ // by converting it to a String in a way which doesn't force a copy),
+ // the buffer size will not decrease, and we will risk wasting a large
+ // amount of memory.
+ // Thanks to Wolfram Esser for spotting this problem.
+ current = new StringBuilder();
+ return result;
+ }
+
// current string
protected internal System.Text.StringBuilder current;
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/SimpleFacetedSearch/TestSimpleFacetedSearch.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/SimpleFacetedSearch/TestSimpleFacetedSearch.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/SimpleFacetedSearch/TestSimpleFacetedSearch.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/SimpleFacetedSearch/TestSimpleFacetedSearch.cs Wed Nov 23 08:38:49 2011
@@ -31,6 +31,7 @@ using Lucene.Net.Store;
using Lucene.Net.Util;
using NUnit.Framework;
+using Version = Lucene.Net.Util.Version;
namespace Lucene.Net.Search
{
@@ -44,7 +45,7 @@ namespace Lucene.Net.Search
public void SetUp()
{
- IndexWriter writer = new IndexWriter(_Dir, new StandardAnalyzer(), true);
+ IndexWriter writer = new IndexWriter(_Dir, new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
AddDoc(writer, "us", "CCN", "politics", "The White House doubles down on social media");
AddDoc(writer, "us", "CCN", "politics", "Senate Dems fail to block filibuster over judicial nominee");
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Analysis/Snowball/TestSnowball.cs
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Analysis/Snowball/TestSnowball.cs?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Analysis/Snowball/TestSnowball.cs (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Analysis/Snowball/TestSnowball.cs Wed Nov 23 08:38:49 2011
@@ -19,129 +19,125 @@
*
*/
-/* ====================================================================
-* The Apache Software License, Version 1.1
-*
-* Copyright (c) 2004 The Apache Software Foundation. All rights
-* reserved.
-*
-* Redistribution and use in source and binary forms, with or without
-* modification, are permitted provided that the following conditions
-* are met:
-*
-* 1. Redistributions of source code must retain the above copyright
-* notice, this list of conditions and the following disclaimer.
-*
-* 2. Redistributions in binary form must reproduce the above copyright
-* notice, this list of conditions and the following disclaimer in
-* the documentation and/or other materials provided with the
-* distribution.
-*
-* 3. The end-user documentation included with the redistribution,
-* if any, must include the following acknowledgment:
-* "This product includes software developed by the
-* Apache Software Foundation (http://www.apache.org/)."
-* Alternately, this acknowledgment may appear in the software itself,
-* if and wherever such third-party acknowledgments normally appear.
-*
-* 4. The names "Apache" and "Apache Software Foundation" and
-* "Apache Lucene" must not be used to endorse or promote products
-* derived from this software without prior written permission. For
-* written permission, please contact apache@apache.org.
-*
-* 5. Products derived from this software may not be called "Apache",
-* "Apache Lucene", nor may "Apache" appear in their name, without
-* prior written permission of the Apache Software Foundation.
-*
-* THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
-* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-* DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
-* ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-* USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
-* OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-* SUCH DAMAGE.
-* ====================================================================
-*
-* This software consists of voluntary contributions made by many
-* individuals on behalf of the Apache Software Foundation. For more
-* information on the Apache Software Foundation, please see
-* <http://www.apache.org/>.
-*/
-
using System;
+using System.IO;
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Index;
using NUnit.Framework;
using Lucene.Net.Analysis;
+using FlagsAttribute = Lucene.Net.Analysis.Tokenattributes.FlagsAttribute;
+using Version = Lucene.Net.Util.Version;
namespace Lucene.Net.Analysis.Snowball
{
- [TestFixture]
- public class TestSnowball
- {
- private class AnonymousClassTokenStream : TokenStream
- {
- public AnonymousClassTokenStream(Token tok, TestSnowball enclosingInstance)
- {
- InitBlock(tok, enclosingInstance);
- }
- private void InitBlock(Token tok, TestSnowball enclosingInstance)
- {
- this.tok = tok;
- this.enclosingInstance = enclosingInstance;
- }
- private Token tok;
- private TestSnowball enclosingInstance;
- public TestSnowball Enclosing_Instance
- {
- get
- {
- return enclosingInstance;
- }
-
- }
- public override Token Next()
- {
- return tok;
- }
- }
-
- public virtual void AssertAnalyzesTo(Analyzer a, System.String input, System.String[] output)
- {
- TokenStream ts = a.TokenStream("dummy", new System.IO.StringReader(input));
- for (int i = 0; i < output.Length; i++)
- {
- Token t = ts.Next();
- Assert.IsNotNull(t);
- Assert.AreEqual(output[i], t.TermText());
- }
- Assert.IsNull(ts.Next());
- ts.Close();
- }
- [Test]
- public virtual void TestEnglish()
- {
- Analyzer a = new SnowballAnalyzer("English");
- AssertAnalyzesTo(a, "he abhorred accents", new System.String[]{"he", "abhor", "accent"});
- }
- [Test]
- public virtual void TestFilterTokens()
- {
- Token tok = new Token("accents", 2, 7, "wrd");
- tok.SetPositionIncrement(3);
-
- SnowballFilter filter = new SnowballFilter(new AnonymousClassTokenStream(tok, this), "English");
-
- Token newtok = filter.Next();
-
- Assert.AreEqual("accent", newtok.TermText());
- Assert.AreEqual(2, newtok.StartOffset());
- Assert.AreEqual(7, newtok.EndOffset());
- Assert.AreEqual("wrd", newtok.Type());
- Assert.AreEqual(3, newtok.GetPositionIncrement());
- }
- }
-}
+ [TestFixture]
+ public class TestSnowball : BaseTokenStreamTestCase
+ {
+ [Test]
+ public void TestEnglish()
+ {
+ Analyzer a = new SnowballAnalyzer(Version.LUCENE_CURRENT, "English");
+ AssertAnalyzesTo(a, "he abhorred accents",
+ new String[] { "he", "abhor", "accent" });
+ }
+
+ [Test]
+ public void TestStopwords()
+ {
+ Analyzer a = new SnowballAnalyzer(Version.LUCENE_CURRENT, "English",
+ StandardAnalyzer.STOP_WORDS_SET);
+ AssertAnalyzesTo(a, "the quick brown fox jumped",
+ new String[] { "quick", "brown", "fox", "jump" });
+ }
+
+ [Test]
+ public void TestReusableTokenStream()
+ {
+ Analyzer a = new SnowballAnalyzer(Version.LUCENE_CURRENT, "English");
+ AssertAnalyzesToReuse(a, "he abhorred accents",
+ new String[] { "he", "abhor", "accent" });
+ AssertAnalyzesToReuse(a, "she abhorred him",
+ new String[] { "she", "abhor", "him" });
+ }
+
+ /**
+ * subclass that acts just like whitespace analyzer for testing
+ */
+ private class SnowballSubclassAnalyzer : SnowballAnalyzer
+ {
+ public SnowballSubclassAnalyzer(String name)
+ : base(Version.LUCENE_CURRENT, name)
+ {
+
+ }
+
+ public override TokenStream TokenStream(String fieldName, TextReader reader)
+ {
+ return new WhitespaceTokenizer(reader);
+ }
+ }
+
+ [Test]
+ public void TestLucene1678BwComp()
+ {
+ Analyzer a = new SnowballSubclassAnalyzer("English");
+ AssertAnalyzesToReuse(a, "he abhorred accents",
+ new String[] { "he", "abhorred", "accents" });
+ }
+
+ [Test]
+ public void TestFilterTokens()
+ {
+ SnowballFilter filter = new SnowballFilter(new TestTokenStream(), "English");
+ TermAttribute termAtt = filter.GetAttribute<TermAttribute>();
+ OffsetAttribute offsetAtt = filter.GetAttribute<OffsetAttribute>();
+ TypeAttribute typeAtt = filter.GetAttribute<TypeAttribute>();
+ PayloadAttribute payloadAtt = filter.GetAttribute<PayloadAttribute>();
+ PositionIncrementAttribute posIncAtt = filter.GetAttribute<PositionIncrementAttribute>();
+ FlagsAttribute flagsAtt = filter.GetAttribute<FlagsAttribute>();
+
+ filter.IncrementToken();
+
+ Assert.AreEqual("accent", termAtt.Term());
+ Assert.AreEqual(2, offsetAtt.StartOffset());
+ Assert.AreEqual(7, offsetAtt.EndOffset());
+ Assert.AreEqual("wrd", typeAtt.Type());
+ Assert.AreEqual(3, posIncAtt.GetPositionIncrement());
+ Assert.AreEqual(77, flagsAtt.GetFlags());
+ Assert.AreEqual(new Payload(new byte[] { 0, 1, 2, 3 }), payloadAtt.GetPayload());
+ }
+
+ private sealed class TestTokenStream : TokenStream
+ {
+ private TermAttribute termAtt;
+ private OffsetAttribute offsetAtt;
+ private TypeAttribute typeAtt;
+ private PayloadAttribute payloadAtt;
+ private PositionIncrementAttribute posIncAtt;
+ private FlagsAttribute flagsAtt;
+
+ internal TestTokenStream()
+ {
+ termAtt = AddAttribute<TermAttribute>();
+ offsetAtt = AddAttribute<OffsetAttribute>();
+ typeAtt = AddAttribute<TypeAttribute>();
+ payloadAtt = AddAttribute<PayloadAttribute>();
+ posIncAtt = AddAttribute<PositionIncrementAttribute>();
+ flagsAtt = AddAttribute<FlagsAttribute>();
+ }
+
+ public override bool IncrementToken()
+ {
+ ClearAttributes();
+ termAtt.SetTermBuffer("accents");
+ offsetAtt.SetOffset(2, 7);
+ typeAtt.SetType("wrd");
+ posIncAtt.SetPositionIncrement(3);
+ payloadAtt.SetPayload(new Payload(new byte[] { 0, 1, 2, 3 }));
+ flagsAtt.SetFlags(77);
+ return true;
+ }
+ }
+ }
+}
\ No newline at end of file
Modified: incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Contrib.Snowball.Test.csproj
URL: http://svn.apache.org/viewvc/incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Contrib.Snowball.Test.csproj?rev=1205326&r1=1205325&r2=1205326&view=diff
==============================================================================
--- incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Contrib.Snowball.Test.csproj (original)
+++ incubator/lucene.net/branches/Lucene.Net.3.0.3/trunk/test/contrib/Snowball/Contrib.Snowball.Test.csproj Wed Nov 23 08:38:49 2011
@@ -19,7 +19,6 @@
under the License.
-->
-
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="Build">
<PropertyGroup>
<ProjectType>Local</ProjectType>
@@ -148,6 +147,10 @@
<Project>{5D4AD9BE-1FFB-41AB-9943-25737971BF57}</Project>
<Name>Lucene.Net</Name>
</ProjectReference>
+ <ProjectReference Include="..\..\core\Lucene.Net.Test.csproj">
+ <Project>{AAF68BCF-F781-45FC-98B3-2B9CEE411E01}</Project>
+ <Name>Lucene.Net.Test</Name>
+ </ProjectReference>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<PropertyGroup>