You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by ma...@apache.org on 2011/03/11 22:24:44 UTC
[lucy-commits] svn commit: r1080766 - in /incubator/lucy/trunk:
clownfish/lib/Clownfish/Binding/ core/Lucy/Analysis/
core/Lucy/Test/Analysis/ modules/analysis/snowstop/devel/
modules/analysis/snowstop/source/ perl/lib/Lucy/ perl/lib/Lucy/Analysis/
perl/lib/Lucy/Docs/...
Author: marvin
Date: Fri Mar 11 21:24:43 2011
New Revision: 1080766
URL: http://svn.apache.org/viewvc?rev=1080766&view=rev
Log:
LUCY-132 snowball_stopfilter.patch
Move Lucy::Analysis::Stopalizer to Lucy::Analysis::SnowballStopFilter.
Added:
incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.c
- copied, changed from r1078955, incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.c
incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.cfh
- copied, changed from r1078955, incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.cfh
incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.c
- copied, changed from r1078955, incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.c
incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.cfh
- copied, changed from r1078955, incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.cfh
incubator/lucy/trunk/perl/lib/Lucy/Analysis/SnowballStopFilter.pm
- copied, changed from r1078955, incubator/lucy/trunk/perl/lib/Lucy/Analysis/Stopalizer.pm
incubator/lucy/trunk/perl/t/155-snowball_stop_filter.t
- copied, changed from r1078955, incubator/lucy/trunk/perl/t/155-stopalizer.t
incubator/lucy/trunk/perl/t/core/155-snowball_stop_filter.t
- copied, changed from r1078955, incubator/lucy/trunk/perl/t/core/155-stopalizer.t
Removed:
incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.c
incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.cfh
incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.c
incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.cfh
incubator/lucy/trunk/perl/lib/Lucy/Analysis/Stopalizer.pm
incubator/lucy/trunk/perl/t/155-stopalizer.t
incubator/lucy/trunk/perl/t/core/155-stopalizer.t
Modified:
incubator/lucy/trunk/clownfish/lib/Clownfish/Binding/Perl.pm
incubator/lucy/trunk/core/Lucy/Analysis/PolyAnalyzer.cfh
incubator/lucy/trunk/core/Lucy/Test/Analysis/TestPolyAnalyzer.c
incubator/lucy/trunk/modules/analysis/snowstop/devel/update_snowstop.pl
incubator/lucy/trunk/modules/analysis/snowstop/source/snowball_stoplists.c
incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod
incubator/lucy/trunk/perl/lib/Lucy/Test.pm
incubator/lucy/trunk/perl/t/601-queryparser.t
incubator/lucy/trunk/perl/t/611-queryparser_syntax.t
Modified: incubator/lucy/trunk/clownfish/lib/Clownfish/Binding/Perl.pm
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/clownfish/lib/Clownfish/Binding/Perl.pm?rev=1080766&r1=1080765&r2=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/clownfish/lib/Clownfish/Binding/Perl.pm (original)
+++ incubator/lucy/trunk/clownfish/lib/Clownfish/Binding/Perl.pm Fri Mar 11 21:24:43 2011
@@ -342,7 +342,8 @@ my %ks_compat = (
'Lucy::Analysis::CaseFolder' => [
qw( KinoSearch::Analysis::CaseFolder KinoSearch::Analysis::LCNormalizer )
],
- 'Lucy::Analysis::Stopalizer' => [qw( KinoSearch::Analysis::Stopalizer )],
+ 'Lucy::Analysis::SnowballStopFilter' =>
+ [qw( KinoSearch::Analysis::Stopalizer )],
'Lucy::Analysis::SnowballStemmer' =>
[qw( KinoSearch::Analysis::Stemmer )],
);
Modified: incubator/lucy/trunk/core/Lucy/Analysis/PolyAnalyzer.cfh
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/core/Lucy/Analysis/PolyAnalyzer.cfh?rev=1080766&r1=1080765&r2=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/core/Lucy/Analysis/PolyAnalyzer.cfh (original)
+++ incubator/lucy/trunk/core/Lucy/Analysis/PolyAnalyzer.cfh Fri Mar 11 21:24:43 2011
@@ -56,7 +56,7 @@ class Lucy::Analysis::PolyAnalyzer
* @param language An ISO code from the list of supported languages.
* @param analyzers An array of Analyzers. The order of the analyzers
* matters. Don't put a SnowballStemmer before a Tokenizer (can't stem whole
- * documents or paragraphs -- just individual words), or a Stopalizer
+ * documents or paragraphs -- just individual words), or a SnowballStopFilter
* after a SnowballStemmer (stemmed words, e.g. "themselv", will not appear in a
* stoplist). In general, the sequence should be: normalize, tokenize,
* stopalize, stem.
Copied: incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.c (from r1078955, incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.c)
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.c?p2=incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.c&p1=incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.c&r1=1078955&r2=1080766&rev=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.c (original)
+++ incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.c Fri Mar 11 21:24:43 2011
@@ -14,24 +14,24 @@
* limitations under the License.
*/
-#define C_LUCY_STOPALIZER
+#define C_LUCY_SNOWBALLSTOPFILTER
#define C_LUCY_TOKEN
#include "Lucy/Util/ToolSet.h"
#include <ctype.h>
-#include "Lucy/Analysis/Stopalizer.h"
+#include "Lucy/Analysis/SnowballStopFilter.h"
#include "Lucy/Analysis/Token.h"
#include "Lucy/Analysis/Inversion.h"
-Stopalizer*
-Stopalizer_new(const CharBuf *language, Hash *stoplist)
+SnowballStopFilter*
+SnowStop_new(const CharBuf *language, Hash *stoplist)
{
- Stopalizer *self = (Stopalizer*)VTable_Make_Obj(STOPALIZER);
- return Stopalizer_init(self, language, stoplist);
+ SnowballStopFilter *self = (SnowballStopFilter*)VTable_Make_Obj(SNOWBALLSTOPFILTER);
+ return SnowStop_init(self, language, stoplist);
}
-Stopalizer*
-Stopalizer_init(Stopalizer *self, const CharBuf *language, Hash *stoplist)
+SnowballStopFilter*
+SnowStop_init(SnowballStopFilter *self, const CharBuf *language, Hash *stoplist)
{
Analyzer_init((Analyzer*)self);
@@ -40,7 +40,7 @@ Stopalizer_init(Stopalizer *self, const
self->stoplist = (Hash*)INCREF(stoplist);
}
else if (language) {
- self->stoplist = Stopalizer_gen_stoplist(language);
+ self->stoplist = SnowStop_gen_stoplist(language);
if (!self->stoplist)
THROW(ERR, "Can't get a stoplist for '%o'", language);
}
@@ -52,14 +52,14 @@ Stopalizer_init(Stopalizer *self, const
}
void
-Stopalizer_destroy(Stopalizer *self)
+SnowStop_destroy(SnowballStopFilter *self)
{
DECREF(self->stoplist);
- SUPER_DESTROY(self, STOPALIZER);
+ SUPER_DESTROY(self, SNOWBALLSTOPFILTER);
}
Inversion*
-Stopalizer_transform(Stopalizer *self, Inversion *inversion)
+SnowStop_transform(SnowballStopFilter *self, Inversion *inversion)
{
Token *token;
Inversion *new_inversion = Inversion_new(NULL);
@@ -75,11 +75,11 @@ Stopalizer_transform(Stopalizer *self, I
}
bool_t
-Stopalizer_equals(Stopalizer *self, Obj *other)
+SnowStop_equals(SnowballStopFilter *self, Obj *other)
{
- Stopalizer *const evil_twin = (Stopalizer*)other;
+ SnowballStopFilter *const evil_twin = (SnowballStopFilter*)other;
if (evil_twin == self) return true;
- if (!Obj_Is_A(other, STOPALIZER)) return false;
+ if (!Obj_Is_A(other, SNOWBALLSTOPFILTER)) return false;
if (!Hash_Equals(evil_twin->stoplist, (Obj*)self->stoplist)) {
return false;
}
@@ -87,25 +87,25 @@ Stopalizer_equals(Stopalizer *self, Obj
}
Hash*
-Stopalizer_gen_stoplist(const CharBuf *language)
+SnowStop_gen_stoplist(const CharBuf *language)
{
CharBuf *lang = CB_new(3);
CB_Cat_Char(lang, tolower(CB_Code_Point_At(language, 0)));
CB_Cat_Char(lang, tolower(CB_Code_Point_At(language, 1)));
const uint8_t **words = NULL;
- if (CB_Equals_Str(lang, "da", 2)) { words = Stopalizer_snow_da; }
- else if (CB_Equals_Str(lang, "de", 2)) { words = Stopalizer_snow_de; }
- else if (CB_Equals_Str(lang, "en", 2)) { words = Stopalizer_snow_en; }
- else if (CB_Equals_Str(lang, "es", 2)) { words = Stopalizer_snow_es; }
- else if (CB_Equals_Str(lang, "fi", 2)) { words = Stopalizer_snow_fi; }
- else if (CB_Equals_Str(lang, "fr", 2)) { words = Stopalizer_snow_fr; }
- else if (CB_Equals_Str(lang, "hu", 2)) { words = Stopalizer_snow_hu; }
- else if (CB_Equals_Str(lang, "it", 2)) { words = Stopalizer_snow_it; }
- else if (CB_Equals_Str(lang, "nl", 2)) { words = Stopalizer_snow_nl; }
- else if (CB_Equals_Str(lang, "no", 2)) { words = Stopalizer_snow_no; }
- else if (CB_Equals_Str(lang, "pt", 2)) { words = Stopalizer_snow_pt; }
- else if (CB_Equals_Str(lang, "ru", 2)) { words = Stopalizer_snow_ru; }
- else if (CB_Equals_Str(lang, "sv", 2)) { words = Stopalizer_snow_sv; }
+ if (CB_Equals_Str(lang, "da", 2)) { words = SnowStop_snow_da; }
+ else if (CB_Equals_Str(lang, "de", 2)) { words = SnowStop_snow_de; }
+ else if (CB_Equals_Str(lang, "en", 2)) { words = SnowStop_snow_en; }
+ else if (CB_Equals_Str(lang, "es", 2)) { words = SnowStop_snow_es; }
+ else if (CB_Equals_Str(lang, "fi", 2)) { words = SnowStop_snow_fi; }
+ else if (CB_Equals_Str(lang, "fr", 2)) { words = SnowStop_snow_fr; }
+ else if (CB_Equals_Str(lang, "hu", 2)) { words = SnowStop_snow_hu; }
+ else if (CB_Equals_Str(lang, "it", 2)) { words = SnowStop_snow_it; }
+ else if (CB_Equals_Str(lang, "nl", 2)) { words = SnowStop_snow_nl; }
+ else if (CB_Equals_Str(lang, "no", 2)) { words = SnowStop_snow_no; }
+ else if (CB_Equals_Str(lang, "pt", 2)) { words = SnowStop_snow_pt; }
+ else if (CB_Equals_Str(lang, "ru", 2)) { words = SnowStop_snow_ru; }
+ else if (CB_Equals_Str(lang, "sv", 2)) { words = SnowStop_snow_sv; }
else {
DECREF(lang);
return NULL;
Copied: incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.cfh (from r1078955, incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.cfh)
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.cfh?p2=incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.cfh&p1=incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.cfh&r1=1078955&r2=1080766&rev=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/core/Lucy/Analysis/Stopalizer.cfh (original)
+++ incubator/lucy/trunk/core/Lucy/Analysis/SnowballStopFilter.cfh Fri Mar 11 21:24:43 2011
@@ -31,7 +31,7 @@ parcel Lucy;
*
* ( "walrus" )
*
- * Stopalizer provides default stoplists for several languages, courtesy of
+ * SnowballStopFilter provides default stoplists for several languages, courtesy of
* the Snowball project (<http://snowball.tartarus.org>), or you may supply
* your own.
*
@@ -54,7 +54,7 @@ parcel Lucy;
* |-----------------------|
*/
-class Lucy::Analysis::Stopalizer
+class Lucy::Analysis::SnowballStopFilter cnick SnowStop
inherits Lucy::Analysis::Analyzer : dumpable {
Hash *stoplist;
@@ -73,15 +73,15 @@ class Lucy::Analysis::Stopalizer
inert const uint8_t** snow_ru;
inert const uint8_t** snow_sv;
- inert incremented Stopalizer*
+ inert incremented SnowballStopFilter*
new(const CharBuf *language = NULL, Hash *stoplist = NULL);
/**
* @param stoplist A hash with stopwords as the keys.
* @param language The ISO code for a supported language.
*/
- public inert Stopalizer*
- init(Stopalizer *self, const CharBuf *language = NULL,
+ public inert SnowballStopFilter*
+ init(SnowballStopFilter *self, const CharBuf *language = NULL,
Hash *stoplist = NULL);
/** Return a Hash with the Snowball stoplist for the supplied language.
@@ -90,16 +90,16 @@ class Lucy::Analysis::Stopalizer
gen_stoplist(const CharBuf *language);
public incremented Inversion*
- Transform(Stopalizer *self, Inversion *inversion);
+ Transform(SnowballStopFilter *self, Inversion *inversion);
public bool_t
- Equals(Stopalizer *self, Obj *other);
+ Equals(SnowballStopFilter *self, Obj *other);
public void
- Destroy(Stopalizer *self);
+ Destroy(SnowballStopFilter *self);
}
-class Lucy::Analysis::Stopalizer::NoCloneHash inherits Lucy::Object::Hash {
+class Lucy::Analysis::SnowballStopFilter::NoCloneHash inherits Lucy::Object::Hash {
inert incremented NoCloneHash*
new(uint32_t capacity = 0);
Modified: incubator/lucy/trunk/core/Lucy/Test/Analysis/TestPolyAnalyzer.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/core/Lucy/Test/Analysis/TestPolyAnalyzer.c?rev=1080766&r1=1080765&r2=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/core/Lucy/Test/Analysis/TestPolyAnalyzer.c (original)
+++ incubator/lucy/trunk/core/Lucy/Test/Analysis/TestPolyAnalyzer.c Fri Mar 11 21:24:43 2011
@@ -22,7 +22,7 @@
#include "Lucy/Test/Analysis/TestPolyAnalyzer.h"
#include "Lucy/Analysis/PolyAnalyzer.h"
#include "Lucy/Analysis/CaseFolder.h"
-#include "Lucy/Analysis/Stopalizer.h"
+#include "Lucy/Analysis/SnowballStopFilter.h"
#include "Lucy/Analysis/SnowballStemmer.h"
#include "Lucy/Analysis/Tokenizer.h"
@@ -57,12 +57,12 @@ test_Dump_Load_and_Equals(TestBatch *bat
static void
test_analysis(TestBatch *batch)
{
- CharBuf *EN = (CharBuf*)ZCB_WRAP_STR("en", 2);
- CharBuf *source_text = CB_newf("Eats, shoots and leaves.");
- CaseFolder *case_folder = CaseFolder_new();
- Tokenizer *tokenizer = Tokenizer_new(NULL);
- Stopalizer *stopalizer = Stopalizer_new(EN, NULL);
- SnowballStemmer *stemmer = SnowStemmer_new(EN);
+ CharBuf *EN = (CharBuf*)ZCB_WRAP_STR("en", 2);
+ CharBuf *source_text = CB_newf("Eats, shoots and leaves.");
+ CaseFolder *case_folder = CaseFolder_new();
+ Tokenizer *tokenizer = Tokenizer_new(NULL);
+ SnowballStopFilter *stopfilter = SnowStop_new(EN, NULL);
+ SnowballStemmer *stemmer = SnowStemmer_new(EN);
{
VArray *analyzers = VA_new(0);
@@ -110,14 +110,14 @@ test_analysis(TestBatch *batch)
VArray *analyzers = VA_new(0);
VA_Push(analyzers, INCREF(case_folder));
VA_Push(analyzers, INCREF(tokenizer));
- VA_Push(analyzers, INCREF(stopalizer));
+ VA_Push(analyzers, INCREF(stopfilter));
PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers);
VArray *expected = VA_new(1);
VA_Push(expected, (Obj*)CB_newf("eats"));
VA_Push(expected, (Obj*)CB_newf("shoots"));
VA_Push(expected, (Obj*)CB_newf("leaves"));
TestUtils_test_analyzer(batch, (Analyzer*)polyanalyzer, source_text,
- expected, "With Stopalizer");
+ expected, "With SnowballStopFilter");
DECREF(expected);
DECREF(polyanalyzer);
DECREF(analyzers);
@@ -127,7 +127,7 @@ test_analysis(TestBatch *batch)
VArray *analyzers = VA_new(0);
VA_Push(analyzers, INCREF(case_folder));
VA_Push(analyzers, INCREF(tokenizer));
- VA_Push(analyzers, INCREF(stopalizer));
+ VA_Push(analyzers, INCREF(stopfilter));
VA_Push(analyzers, INCREF(stemmer));
PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers);
VArray *expected = VA_new(1);
@@ -142,7 +142,7 @@ test_analysis(TestBatch *batch)
}
DECREF(stemmer);
- DECREF(stopalizer);
+ DECREF(stopfilter);
DECREF(tokenizer);
DECREF(case_folder);
DECREF(source_text);
Copied: incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.c (from r1078955, incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.c)
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.c?p2=incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.c&p1=incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.c&r1=1078955&r2=1080766&rev=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.c (original)
+++ incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.c Fri Mar 11 21:24:43 2011
@@ -14,19 +14,19 @@
* limitations under the License.
*/
-#define C_LUCY_TESTSTOPALIZER
+#define C_LUCY_TESTSNOWBALLSTOPFILTER
#include "Lucy/Util/ToolSet.h"
#include <stdarg.h>
#include "Lucy/Test.h"
-#include "Lucy/Test/Analysis/TestStopalizer.h"
-#include "Lucy/Analysis/Stopalizer.h"
+#include "Lucy/Test/Analysis/TestSnowballStopFilter.h"
+#include "Lucy/Analysis/SnowballStopFilter.h"
-static Stopalizer*
-S_make_stopalizer(void *unused, ...)
+static SnowballStopFilter*
+S_make_stopfilter(void *unused, ...)
{
va_list args;
- Stopalizer *self = (Stopalizer*)VTable_Make_Obj(STOPALIZER);
+ SnowballStopFilter *self = (SnowballStopFilter*)VTable_Make_Obj(SNOWBALLSTOPFILTER);
Hash *stoplist = Hash_new(0);
char *stopword;
@@ -36,7 +36,7 @@ S_make_stopalizer(void *unused, ...)
}
va_end(args);
- self = Stopalizer_init(self, NULL, stoplist);
+ self = SnowStop_init(self, NULL, stoplist);
DECREF(stoplist);
return self;
}
@@ -44,22 +44,22 @@ S_make_stopalizer(void *unused, ...)
static void
test_Dump_Load_and_Equals(TestBatch *batch)
{
- Stopalizer *stopalizer
- = S_make_stopalizer(NULL, "foo", "bar", "baz", NULL);
- Stopalizer *other = S_make_stopalizer(NULL, "foo", "bar", NULL);
- Obj *dump = Stopalizer_Dump(stopalizer);
- Obj *other_dump = Stopalizer_Dump(other);
- Stopalizer *clone = (Stopalizer*)Stopalizer_Load(other, dump);
- Stopalizer *other_clone = (Stopalizer*)Stopalizer_Load(other, other_dump);
+ SnowballStopFilter *stopfilter = S_make_stopfilter(NULL, "foo", "bar",
+ "baz", NULL);
+ SnowballStopFilter *other = S_make_stopfilter(NULL, "foo", "bar", NULL);
+ Obj *dump = SnowStop_Dump(stopfilter);
+ Obj *other_dump = SnowStop_Dump(other);
+ SnowballStopFilter *clone = (SnowballStopFilter*)SnowStop_Load(other, dump);
+ SnowballStopFilter *other_clone = (SnowballStopFilter*)SnowStop_Load(other, other_dump);
- TEST_FALSE(batch, Stopalizer_Equals(stopalizer,
+ TEST_FALSE(batch, SnowStop_Equals(stopfilter,
(Obj*)other), "Equals() false with different stoplist");
- TEST_TRUE(batch, Stopalizer_Equals(stopalizer,
+ TEST_TRUE(batch, SnowStop_Equals(stopfilter,
(Obj*)clone), "Dump => Load round trip");
- TEST_TRUE(batch, Stopalizer_Equals(other,
+ TEST_TRUE(batch, SnowStop_Equals(other,
(Obj*)other_clone), "Dump => Load round trip");
- DECREF(stopalizer);
+ DECREF(stopfilter);
DECREF(dump);
DECREF(clone);
DECREF(other);
@@ -68,7 +68,7 @@ test_Dump_Load_and_Equals(TestBatch *bat
}
void
-TestStopalizer_run_tests()
+TestSnowStop_run_tests()
{
TestBatch *batch = TestBatch_new(3);
Copied: incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.cfh (from r1078955, incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.cfh)
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.cfh?p2=incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.cfh&p1=incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.cfh&r1=1078955&r2=1080766&rev=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/core/Lucy/Test/Analysis/TestStopalizer.cfh (original)
+++ incubator/lucy/trunk/core/Lucy/Test/Analysis/TestSnowballStopFilter.cfh Fri Mar 11 21:24:43 2011
@@ -16,7 +16,7 @@
parcel Lucy;
-inert class Lucy::Test::Analysis::TestStopalizer {
+inert class Lucy::Test::Analysis::TestSnowballStopFilter cnick TestSnowStop {
inert void
run_tests();
}
Modified: incubator/lucy/trunk/modules/analysis/snowstop/devel/update_snowstop.pl
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/modules/analysis/snowstop/devel/update_snowstop.pl?rev=1080766&r1=1080765&r2=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/modules/analysis/snowstop/devel/update_snowstop.pl (original)
+++ incubator/lucy/trunk/modules/analysis/snowstop/devel/update_snowstop.pl Fri Mar 11 21:24:43 2011
@@ -43,7 +43,7 @@ print $out_fh <<'END_STUFF';
* project.
*/
-#include "Lucy/Analysis/Stopalizer.h"
+#include "Lucy/Analysis/SnowballStopFilter.h"
END_STUFF
@@ -90,7 +90,7 @@ for my $iso ( sort keys %languages ) {
static const char *words_${iso}[] = {
$wrapped
};
-const uint8_t **lucy_Stopalizer_snow_${iso} = (const uint8_t**)words_$iso;
+const uint8_t **lucy_SnowStop_snow_${iso} = (const uint8_t**)words_$iso;
END_STUFF
}
Modified: incubator/lucy/trunk/modules/analysis/snowstop/source/snowball_stoplists.c
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/modules/analysis/snowstop/source/snowball_stoplists.c?rev=1080766&r1=1080765&r2=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/modules/analysis/snowstop/source/snowball_stoplists.c (original)
+++ incubator/lucy/trunk/modules/analysis/snowstop/source/snowball_stoplists.c Fri Mar 11 21:24:43 2011
@@ -4,7 +4,7 @@
* project.
*/
-#include "Lucy/Analysis/Stopalizer.h"
+#include "Lucy/Analysis/SnowballStopFilter.h"
static const char *words_da[] = {
"og", "i", "jeg", "det", "at", "en", "den", "til", "er", "som",
@@ -19,7 +19,7 @@ static const char *words_da[] = {
"hvis", "din", "nogle", "hos", "blive", "mange", "ad", "bliver",
"hendes", "v\303\246ret", "thi", "jer", "s\303\245dan", NULL
};
-const uint8_t **lucy_Stopalizer_snow_da = (const uint8_t**)words_da;
+const uint8_t **lucy_SnowStop_snow_da = (const uint8_t**)words_da;
static const char *words_de[] = {
"aber", "alle", "allem", "allen", "aller", "alles", "als", "also", "am",
@@ -54,7 +54,7 @@ static const char *words_de[] = {
"w\303\274rde", "w\303\274rden", "zu", "zum", "zur", "zwar", "zwischen",
NULL
};
-const uint8_t **lucy_Stopalizer_snow_de = (const uint8_t**)words_de;
+const uint8_t **lucy_SnowStop_snow_de = (const uint8_t**)words_de;
static const char *words_en[] = {
"i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you",
@@ -79,7 +79,7 @@ static const char *words_en[] = {
"each", "few", "more", "most", "other", "some", "such", "no", "nor",
"not", "only", "own", "same", "so", "than", "too", "very", NULL
};
-const uint8_t **lucy_Stopalizer_snow_en = (const uint8_t**)words_en;
+const uint8_t **lucy_SnowStop_snow_en = (const uint8_t**)words_en;
static const char *words_es[] = {
"de", "la", "que", "el", "en", "y", "a", "los", "del", "se", "las",
@@ -137,7 +137,7 @@ static const char *words_es[] = {
"tuviesen", "teniendo", "tenido", "tenida", "tenidos", "tenidas",
"tened", NULL
};
-const uint8_t **lucy_Stopalizer_snow_es = (const uint8_t**)words_es;
+const uint8_t **lucy_SnowStop_snow_es = (const uint8_t**)words_es;
static const char *words_fi[] = {
"olla", "olen", "olet", "on", "olemme", "olette", "ovat", "ole", "oli",
@@ -185,7 +185,7 @@ static const char *words_fi[] = {
"sill\303\244", "tai", "vaan", "vai", "vaikka", "kanssa", "mukaan",
"noin", "poikki", "yli", "kun", "niin", "nyt", "itse", NULL
};
-const uint8_t **lucy_Stopalizer_snow_fi = (const uint8_t**)words_fi;
+const uint8_t **lucy_SnowStop_snow_fi = (const uint8_t**)words_fi;
static const char *words_fr[] = {
"au", "aux", "avec", "ce", "ces", "dans", "de", "des", "du", "elle",
@@ -211,7 +211,7 @@ static const char *words_fr[] = {
"ils", "les", "leurs", "quel", "quels", "quelle", "quelles", "sans",
"soi", NULL
};
-const uint8_t **lucy_Stopalizer_snow_fr = (const uint8_t**)words_fr;
+const uint8_t **lucy_SnowStop_snow_fr = (const uint8_t**)words_fr;
static const char *words_hu[] = {
"a", "ahogy", "ahol", "aki", "akik", "akkor", "alatt", "\303\241ltal",
@@ -248,7 +248,7 @@ static const char *words_hu[] = {
"vannak", "volt", "voltam", "voltak", "voltunk", "vissza", "vele",
"viszont", "volna", NULL
};
-const uint8_t **lucy_Stopalizer_snow_hu = (const uint8_t**)words_hu;
+const uint8_t **lucy_SnowStop_snow_hu = (const uint8_t**)words_hu;
static const char *words_it[] = {
"ad", "al", "allo", "ai", "agli", "all", "agl", "alla", "alle", "con",
@@ -289,7 +289,7 @@ static const char *words_it[] = {
"stette", "stemmo", "steste", "stettero", "stessi", "stesse", "stessimo",
"stessero", "stando", NULL
};
-const uint8_t **lucy_Stopalizer_snow_it = (const uint8_t**)words_it;
+const uint8_t **lucy_SnowStop_snow_it = (const uint8_t**)words_it;
static const char *words_nl[] = {
"de", "en", "van", "ik", "te", "dat", "die", "in", "een", "hij", "het",
@@ -304,7 +304,7 @@ static const char *words_nl[] = {
"wezen", "kunnen", "ons", "zelf", "tegen", "na", "reeds", "wil", "kon",
"niets", "uw", "iemand", "geweest", "andere", NULL
};
-const uint8_t **lucy_Stopalizer_snow_nl = (const uint8_t**)words_nl;
+const uint8_t **lucy_SnowStop_snow_nl = (const uint8_t**)words_nl;
static const char *words_no[] = {
"og", "i", "jeg", "det", "at", "en", "et", "den", "til", "er", "som",
@@ -330,7 +330,7 @@ static const char *words_no[] = {
"sidan", "so", "somt", "somme", "um", "upp", "vere", "vore", "verte",
"vort", "varte", "vart", NULL
};
-const uint8_t **lucy_Stopalizer_snow_no = (const uint8_t**)words_no;
+const uint8_t **lucy_SnowStop_snow_no = (const uint8_t**)words_no;
static const char *words_pt[] = {
"de", "a", "o", "que", "e", "do", "da", "em", "um", "para", "com",
@@ -365,7 +365,7 @@ static const char *words_pt[] = {
"tivermos", "tiverem", "terei", "ter\303\241", "teremos", "ter\303\243o",
"teria", "ter\303\255amos", "teriam", NULL
};
-const uint8_t **lucy_Stopalizer_snow_pt = (const uint8_t**)words_pt;
+const uint8_t **lucy_SnowStop_snow_pt = (const uint8_t**)words_pt;
static const char *words_ru[] = {
"\320\270", "\320\262", "\320\262\320\276", "\320\275\320\265",
@@ -465,7 +465,7 @@ static const char *words_ru[] = {
"\320\262\321\201\321\216", "\320\274\320\265\320\266\320\264\321\203",
NULL
};
-const uint8_t **lucy_Stopalizer_snow_ru = (const uint8_t**)words_ru;
+const uint8_t **lucy_SnowStop_snow_ru = (const uint8_t**)words_ru;
static const char *words_sv[] = {
"och", "det", "att", "i", "en", "jag", "hon", "som", "han", "p\303\245",
@@ -484,5 +484,5 @@ static const char *words_sv[] = {
"ditt", "vem", "vilket", "sitta", "s\303\245dana", "vart", "dina",
"vars", "v\303\245rt", "v\303\245ra", "ert", "era", "vilkas", NULL
};
-const uint8_t **lucy_Stopalizer_snow_sv = (const uint8_t**)words_sv;
+const uint8_t **lucy_SnowStop_snow_sv = (const uint8_t**)words_sv;
Copied: incubator/lucy/trunk/perl/lib/Lucy/Analysis/SnowballStopFilter.pm (from r1078955, incubator/lucy/trunk/perl/lib/Lucy/Analysis/Stopalizer.pm)
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/lib/Lucy/Analysis/SnowballStopFilter.pm?p2=incubator/lucy/trunk/perl/lib/Lucy/Analysis/SnowballStopFilter.pm&p1=incubator/lucy/trunk/perl/lib/Lucy/Analysis/Stopalizer.pm&r1=1078955&r2=1080766&rev=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/lib/Lucy/Analysis/Stopalizer.pm (original)
+++ incubator/lucy/trunk/perl/lib/Lucy/Analysis/SnowballStopFilter.pm Fri Mar 11 21:24:43 2011
@@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-package Lucy::Analysis::Stopalizer;
+package Lucy::Analysis::SnowballStopFilter;
use Lucy;
1;
@@ -23,28 +23,28 @@ __END__
__BINDING__
my $synopsis = <<'END_SYNOPSIS';
- my $stopalizer = Lucy::Analysis::Stopalizer->new(
+ my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
language => 'fr',
);
my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
- analyzers => [ $case_folder, $tokenizer, $stopalizer, $stemmer ],
+ analyzers => [ $case_folder, $tokenizer, $stopfilter, $stemmer ],
);
END_SYNOPSIS
my $constructor = <<'END_CONSTRUCTOR';
- my $stopalizer = Lucy::Analysis::Stopalizer->new(
+ my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
language => 'de',
);
# or...
- my $stopalizer = Lucy::Analysis::Stopalizer->new(
+ my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
stoplist => \%stoplist,
);
END_CONSTRUCTOR
Clownfish::Binding::Perl::Class->register(
parcel => "Lucy",
- class_name => "Lucy::Analysis::Stopalizer",
+ class_name => "Lucy::Analysis::SnowballStopFilter",
bind_constructors => ["new"],
make_pod => {
synopsis => $synopsis,
Modified: incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod?rev=1080766&r1=1080765&r2=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod (original)
+++ incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod Fri Mar 11 21:24:43 2011
@@ -52,14 +52,14 @@ C<< language => 'en' >> is equivalent to
);
You can add or subtract Analyzers from there if you like. Try adding a fourth
-Analyzer, a Stopalizer for suppressing "stopwords" like "the", "if",
+Analyzer, a SnowballStopFilter for suppressing "stopwords" like "the", "if",
and "maybe".
- my $stopalizer = Lucy::Analysis::Stopalizer->new(
+ my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
language => 'en',
);
my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
- analyzers => [ $case_folder, $tokenizer, $stopalizer, $stemmer ],
+ analyzers => [ $case_folder, $tokenizer, $stopfilter, $stemmer ],
);
Also, try removing the SnowballStemmer.
Modified: incubator/lucy/trunk/perl/lib/Lucy/Test.pm
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/lib/Lucy/Test.pm?rev=1080766&r1=1080765&r2=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/lib/Lucy/Test.pm (original)
+++ incubator/lucy/trunk/perl/lib/Lucy/Test.pm Fri Mar 11 21:24:43 2011
@@ -52,8 +52,8 @@ PPCODE:
else if (strEQ(package, "TestPolyAnalyzer")) {
lucy_TestPolyAnalyzer_run_tests();
}
- else if (strEQ(package, "TestStopalizer")) {
- lucy_TestStopalizer_run_tests();
+ else if (strEQ(package, "TestSnowballStopFilter")) {
+ lucy_TestSnowStop_run_tests();
}
else if (strEQ(package, "TestSnowStemmer")) {
lucy_TestSnowStemmer_run_tests();
Copied: incubator/lucy/trunk/perl/t/155-snowball_stop_filter.t (from r1078955, incubator/lucy/trunk/perl/t/155-stopalizer.t)
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/t/155-snowball_stop_filter.t?p2=incubator/lucy/trunk/perl/t/155-snowball_stop_filter.t&p1=incubator/lucy/trunk/perl/t/155-stopalizer.t&r1=1078955&r2=1080766&rev=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/t/155-stopalizer.t (original)
+++ incubator/lucy/trunk/perl/t/155-snowball_stop_filter.t Fri Mar 11 21:24:43 2011
@@ -20,11 +20,11 @@ use lib 'buildlib';
use Test::More tests => 6;
use Lucy::Test::TestUtils qw( test_analyzer );
-my $stopalizer = Lucy::Analysis::Stopalizer->new( language => 'en' );
-test_analyzer( $stopalizer, 'the', [], "single stopword stopalized" );
+my $stopfilter = Lucy::Analysis::SnowballStopFilter->new( language => 'en' );
+test_analyzer( $stopfilter, 'the', [], "single stopword stopalized" );
my $tokenizer = Lucy::Analysis::Tokenizer->new;
my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
- analyzers => [ $tokenizer, $stopalizer ], );
+ analyzers => [ $tokenizer, $stopfilter ], );
test_analyzer( $polyanalyzer, 'i am the walrus',
['walrus'], "multiple stopwords stopalized" );
Modified: incubator/lucy/trunk/perl/t/601-queryparser.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/t/601-queryparser.t?rev=1080766&r1=1080765&r2=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/t/601-queryparser.t (original)
+++ incubator/lucy/trunk/perl/t/601-queryparser.t Fri Mar 11 21:24:43 2011
@@ -38,10 +38,10 @@ sub new {
my $self = shift->SUPER::new(@_);
my $whitespace_tokenizer
= Lucy::Analysis::Tokenizer->new( token_re => qr/\S+/ );
- my $stopalizer
- = Lucy::Analysis::Stopalizer->new( stoplist => { x => 1 } );
+ my $stopfilter
+ = Lucy::Analysis::SnowballStopFilter->new( stoplist => { x => 1 } );
my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
- analyzers => [ $whitespace_tokenizer, $stopalizer, ], );
+ analyzers => [ $whitespace_tokenizer, $stopfilter, ], );
my $type
= Lucy::Plan::FullTextType->new( analyzer => $polyanalyzer, );
$self->spec_field( name => 'content', type => $type );
Modified: incubator/lucy/trunk/perl/t/611-queryparser_syntax.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/t/611-queryparser_syntax.t?rev=1080766&r1=1080765&r2=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/t/611-queryparser_syntax.t (original)
+++ incubator/lucy/trunk/perl/t/611-queryparser_syntax.t Fri Mar 11 21:24:43 2011
@@ -27,10 +27,10 @@ sub new {
my $tokenizer = Lucy::Analysis::Tokenizer->new( pattern => '\S+' );
my $wordchar_tokenizer
= Lucy::Analysis::Tokenizer->new( pattern => '\w+', );
- my $stopalizer
- = Lucy::Analysis::Stopalizer->new( stoplist => { x => 1 } );
+ my $stopfilter
+ = Lucy::Analysis::SnowballStopFilter->new( stoplist => { x => 1 } );
my $fancy_analyzer = Lucy::Analysis::PolyAnalyzer->new(
- analyzers => [ $wordchar_tokenizer, $stopalizer, ], );
+ analyzers => [ $wordchar_tokenizer, $stopfilter, ], );
my $plain = Lucy::Plan::FullTextType->new( analyzer => $tokenizer );
my $fancy
Copied: incubator/lucy/trunk/perl/t/core/155-snowball_stop_filter.t (from r1078955, incubator/lucy/trunk/perl/t/core/155-stopalizer.t)
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/t/core/155-snowball_stop_filter.t?p2=incubator/lucy/trunk/perl/t/core/155-snowball_stop_filter.t&p1=incubator/lucy/trunk/perl/t/core/155-stopalizer.t&r1=1078955&r2=1080766&rev=1080766&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/t/core/155-stopalizer.t (original)
+++ incubator/lucy/trunk/perl/t/core/155-snowball_stop_filter.t Fri Mar 11 21:24:43 2011
@@ -17,5 +17,5 @@ use strict;
use warnings;
use Lucy::Test;
-Lucy::Test::run_tests("TestStopalizer");
+Lucy::Test::run_tests("TestSnowballStopFilter");