You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by nw...@apache.org on 2012/02/08 15:50:05 UTC
[lucy-commits] svn commit: r1241917 - in /incubator/lucy/trunk/perl:
buildlib/Lucy/Build/Binding/ lib/ lib/Lucy/Docs/Cookbook/
lib/Lucy/Docs/Tutorial/ sample/
Author: nwellnhof
Date: Wed Feb 8 14:50:04 2012
New Revision: 1241917
URL: http://svn.apache.org/viewvc?rev=1241917&view=rev
Log:
Replace PolyAnalyzer with EasyAnalyzer in docs
Modified:
incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Analysis.pm
incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Plan.pm
incubator/lucy/trunk/perl/lib/Lucy.pod
incubator/lucy/trunk/perl/lib/Lucy/Docs/Cookbook/CustomQueryParser.pod
incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod
incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/BeyondSimple.pod
incubator/lucy/trunk/perl/sample/indexer.pl
Modified: incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Analysis.pm
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Analysis.pm?rev=1241917&r1=1241916&r2=1241917&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Analysis.pm (original)
+++ incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Analysis.pm Wed Feb 8 14:50:04 2012
@@ -186,7 +186,7 @@ sub bind_polyanalyzer {
my $synopsis = <<'END_SYNOPSIS';
my $schema = Lucy::Plan::Schema->new;
my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
- language => 'en',
+ analyzers => \@analyzers,
);
my $type = Lucy::Plan::FullTextType->new(
analyzer => $polyanalyzer,
@@ -195,17 +195,11 @@ sub bind_polyanalyzer {
$schema->spec_field( name => 'content', type => $type );
END_SYNOPSIS
my $constructor = <<'END_CONSTRUCTOR';
- my $analyzer = Lucy::Analysis::PolyAnalyzer->new(
- language => 'es',
- );
-
- # or...
-
- my $case_folder = Lucy::Analysis::CaseFolder->new;
my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $normalizer = Lucy::Analysis::Normalizer->new;
my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'en' );
my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
- analyzers => [ $case_folder, $whitespace_tokenizer, $stemmer, ], );
+ analyzers => [ $tokenizer, $normalizer, $stemmer, ], );
END_CONSTRUCTOR
$pod_spec->set_synopsis($synopsis);
$pod_spec->add_constructor( alias => 'new', sample => $constructor );
Modified: incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Plan.pm
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Plan.pm?rev=1241917&r1=1241916&r2=1241917&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Plan.pm (original)
+++ incubator/lucy/trunk/perl/buildlib/Lucy/Build/Binding/Plan.pm Wed Feb 8 14:50:04 2012
@@ -201,11 +201,11 @@ sub bind_fulltexttype {
my $pod_spec = Clownfish::CFC::Binding::Perl::Pod->new;
my $synopsis = <<'END_SYNOPSIS';
- my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ my $easyanalyzer = Lucy::Analysis::EasyAnalyzer->new(
language => 'en',
);
my $type = Lucy::Plan::FullTextType->new(
- analyzer => $polyanalyzer,
+ analyzer => $easyanalyzer,
);
my $schema = Lucy::Plan::Schema->new;
$schema->spec_field( name => 'title', type => $type );
@@ -271,14 +271,14 @@ sub bind_schema {
my $synopsis = <<'END_SYNOPSIS';
use Lucy::Plan::Schema;
use Lucy::Plan::FullTextType;
- use Lucy::Analysis::PolyAnalyzer;
+ use Lucy::Analysis::EasyAnalyzer;
my $schema = Lucy::Plan::Schema->new;
- my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ my $easyanalyzer = Lucy::Analysis::EasyAnalyzer->new(
language => 'en',
);
my $type = Lucy::Plan::FullTextType->new(
- analyzer => $polyanalyzer,
+ analyzer => $easyanalyzer,
);
$schema->spec_field( name => 'title', type => $type );
$schema->spec_field( name => 'content', type => $type );
Modified: incubator/lucy/trunk/perl/lib/Lucy.pod
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/lib/Lucy.pod?rev=1241917&r1=1241916&r2=1241917&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/lib/Lucy.pod (original)
+++ incubator/lucy/trunk/perl/lib/Lucy.pod Wed Feb 8 14:50:04 2012
@@ -29,16 +29,16 @@ First, plan out your index structure, cr
use Lucy::Index::Indexer;
use Lucy::Plan::Schema;
- use Lucy::Analysis::PolyAnalyzer;
+ use Lucy::Analysis::EasyAnalyzer;
use Lucy::Plan::FullTextType;
# Create a Schema which defines index fields.
my $schema = Lucy::Plan::Schema->new;
- my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ my $easyanalyzer = Lucy::Analysis::EasyAnalyzer->new(
language => 'en',
);
my $type = Lucy::Plan::FullTextType->new(
- analyzer => $polyanalyzer,
+ analyzer => $easyanalyzer,
);
$schema->spec_field( name => 'title', type => $type );
$schema->spec_field( name => 'content', type => $type );
@@ -162,7 +162,7 @@ L<Lucy::Search::IndexSearcher> - Search
=item *
-L<Lucy::Analysis::PolyAnalyzer> - A one-size-fits-all parser/tokenizer.
+L<Lucy::Analysis::EasyAnalyzer> - A one-size-fits-all parser/tokenizer.
=back
Modified: incubator/lucy/trunk/perl/lib/Lucy/Docs/Cookbook/CustomQueryParser.pod
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/lib/Lucy/Docs/Cookbook/CustomQueryParser.pod?rev=1241917&r1=1241916&r2=1241917&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/lib/Lucy/Docs/Cookbook/CustomQueryParser.pod (original)
+++ incubator/lucy/trunk/perl/lib/Lucy/Docs/Cookbook/CustomQueryParser.pod Wed Feb 8 14:50:04 2012
@@ -37,7 +37,7 @@ Later, we'll add support for trailing wi
Our initial parser implentation will generate queries against a single fixed
field, "content", and it will analyze text using a fixed choice of English
-PolyAnalyzer. We won't subclass Lucy::Search::QueryParser just yet.
+EasyAnalyzer. We won't subclass Lucy::Search::QueryParser just yet.
package FlatQueryParser;
use Lucy::Search::TermQuery;
@@ -46,7 +46,7 @@ PolyAnalyzer. We won't subclass Lucy::S
use Carp;
sub new {
- my $analyzer = Lucy::Analysis::PolyAnalyzer->new(
+ my $analyzer = Lucy::Analysis::EasyAnalyzer->new(
language => 'en',
);
return bless {
@@ -96,9 +96,9 @@ single token and splits on whitespace ev
}
The main parsing routine creates an array of tokens by calling _tokenize(),
-runs the tokens through through the PolyAnalyzer, creates TermQuery or
+runs the tokens through through the EasyAnalyzer, creates TermQuery or
PhraseQuery objects according to how many tokens emerge from the
-PolyAnalyzer's split() method, and adds each of the sub-queries to the primary
+EasyAnalyzer's split() method, and adds each of the sub-queries to the primary
ORQuery.
sub parse {
@@ -147,7 +147,7 @@ routines to accomplish that expansion, w
and take advantage of some of its existing methods.
Our first parser implementation had the "content" field name and the choice of
-English PolyAnalyzer hard-coded for simplicity, but we don't need to do that
+English EasyAnalyzer hard-coded for simplicity, but we don't need to do that
once we subclass Lucy::Search::QueryParser. QueryParser's constructor --
which we will inherit, allowing us to eliminate our own constructor --
requires a Schema which conveys field
@@ -212,7 +212,7 @@ parent class implementation on TermQuery
}
Ordinarily, those asterisks would have been stripped when running tokens
-through the PolyAnalyzer -- query strings containing "foo*" would produce
+through the EasyAnalyzer -- query strings containing "foo*" would produce
TermQueries for the term "foo". Our override intercepts tokens with trailing
asterisks and processes them as PrefixQueries before C<SUPER::expand_leaf> can
discard them, so that a search for "foo*" can match "food", "foosball", and so
Modified: incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod?rev=1241917&r1=1241916&r2=1241917&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod (original)
+++ incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/Analysis.pod Wed Feb 8 14:50:04 2012
@@ -19,7 +19,7 @@ Lucy::Docs::Tutorial::Analysis - How to
=head1 DESCRIPTION
-Try swapping out the PolyAnalyzer in our Schema for a RegexTokenizer:
+Try swapping out the EasyAnalyzer in our Schema for a RegexTokenizer:
my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
my $type = Lucy::Plan::FullTextType->new(
@@ -29,26 +29,26 @@ Try swapping out the PolyAnalyzer in our
Search for C<senate>, C<Senate>, and C<Senator> before and after making the
change and re-indexing.
-Under PolyAnalyzer, the results are identical for all three searches, but
+Under EasyAnalyzer, the results are identical for all three searches, but
under RegexTokenizer, searches are case-sensitive, and the result sets for
C<Senate> and C<Senator> are distinct.
-=head2 PolyAnalyzer
+=head2 EasyAnalyzer
-What's happening is that PolyAnalyzer is performing more aggressive processing
+What's happening is that EasyAnalyzer is performing more aggressive processing
than RegexTokenizer. In addition to tokenizing, it's also converting all text to
lower case so that searches are case-insensitive, and using a "stemming"
algorithm to reduce related words to a common stem (C<senat>, in this case).
-PolyAnalyzer is actually multiple Analyzers wrapped up in a single package.
-In this case, it's three-in-one, since specifying a PolyAnalyzer with
+EasyAnalyzer is actually multiple Analyzers wrapped up in a single package.
+In this case, it's three-in-one, since specifying a EasyAnalyzer with
C<< language => 'en' >> is equivalent to this snippet:
- my $case_folder = Lucy::Analysis::CaseFolder->new;
my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $normalizer = Lucy::Analysis::Normalizer->new;
my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'en' );
my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
- analyzers => [ $case_folder, $tokenizer, $stemmer ],
+ analyzers => [ $tokenizer, $normalizer, $stemmer ],
);
You can add or subtract Analyzers from there if you like. Try adding a fourth
@@ -59,16 +59,16 @@ and "maybe".
language => 'en',
);
my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
- analyzers => [ $case_folder, $tokenizer, $stopfilter, $stemmer ],
+ analyzers => [ $tokenizer, $normalizer, $stopfilter, $stemmer ],
);
Also, try removing the SnowballStemmer.
my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
- analyzers => [ $case_folder, $tokenizer ],
+ analyzers => [ $tokenizer, $normalizer ],
);
-The original choice of a stock English PolyAnalyzer probably still yields the
+The original choice of a stock English EasyAnalyzer probably still yields the
best results for this document collection, but you get the idea: sometimes you
want a different Analyzer.
Modified: incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/BeyondSimple.pod
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/BeyondSimple.pod?rev=1241917&r1=1241916&r2=1241917&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/BeyondSimple.pod (original)
+++ incubator/lucy/trunk/perl/lib/Lucy/Docs/Tutorial/BeyondSimple.pod Wed Feb 8 14:50:04 2012
@@ -41,7 +41,7 @@ L<Lucy::Plan::FullTextType> - Field type
=item *
-L<Lucy::Analysis::PolyAnalyzer> - A one-size-fits-all parser/tokenizer.
+L<Lucy::Analysis::EasyAnalyzer> - A one-size-fits-all parser/tokenizer.
=item *
@@ -63,7 +63,7 @@ After we load our modules...
use Lucy::Plan::Schema;
use Lucy::Plan::FullTextType;
- use Lucy::Analysis::PolyAnalyzer;
+ use Lucy::Analysis::EasyAnalyzer;
use Lucy::Index::Indexer;
... the first item we're going need is a L<Schema|Lucy::Plan::Schema>.
@@ -73,11 +73,11 @@ they're defined. We'll start off with t
# Create Schema.
my $schema = Lucy::Plan::Schema->new;
- my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ my $easyanalyzer = Lucy::Analysis::EasyAnalyzer->new(
language => 'en',
);
my $type = Lucy::Plan::FullTextType->new(
- analyzer => $polyanalyzer,
+ analyzer => $easyanalyzer,
);
$schema->spec_field( name => 'title', type => $type );
$schema->spec_field( name => 'content', type => $type );
Modified: incubator/lucy/trunk/perl/sample/indexer.pl
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/perl/sample/indexer.pl?rev=1241917&r1=1241916&r2=1241917&view=diff
==============================================================================
--- incubator/lucy/trunk/perl/sample/indexer.pl (original)
+++ incubator/lucy/trunk/perl/sample/indexer.pl Wed Feb 8 14:50:04 2012
@@ -25,19 +25,19 @@ my $uscon_source = '/usr/local/apache2/
use File::Spec::Functions qw( catfile );
use Lucy::Plan::Schema;
use Lucy::Plan::FullTextType;
-use Lucy::Analysis::PolyAnalyzer;
+use Lucy::Analysis::EasyAnalyzer;
use Lucy::Index::Indexer;
# Create Schema.
my $schema = Lucy::Plan::Schema->new;
-my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+my $easyanalyzer = Lucy::Analysis::EasyAnalyzer->new(
language => 'en',
);
my $title_type = Lucy::Plan::FullTextType->new(
- analyzer => $polyanalyzer,
+ analyzer => $easyanalyzer,
);
my $content_type = Lucy::Plan::FullTextType->new(
- analyzer => $polyanalyzer,
+ analyzer => $easyanalyzer,
highlightable => 1,
);
my $url_type = Lucy::Plan::StringType->new( indexed => 0, );