You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucy.apache.org by mi...@apache.org on 2010/10/14 10:48:40 UTC
[lucy-commits] svn commit: r1022422 [40/43] - in /incubator/lucy/trunk/import: ./
charmonizer/ charmonizer/src/ charmonizer/src/Charmonizer/
charmonizer/src/Charmonizer/Core/ charmonizer/src/Charmonizer/Probe/
charmonizer/src/Charmonizer/Test/ clownfish/ clownfish/li...
Added: incubator/lucy/trunk/import/perl/t/216-schema.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/216-schema.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/216-schema.t (added)
+++ incubator/lucy/trunk/import/perl/t/216-schema.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,32 @@
+use strict;
+use warnings;
+
+use lib 'buildlib';
+use KinoSearch::Test;
+
+package main;
+use Test::More tests => 3;
+
+require KinoSearch::Schema;
+my $old_schema = KinoSearch::Schema->new;
+my $new_schema = KinoSearch::Plan::Schema->new;
+
+$old_schema->eat($new_schema);
+$new_schema->eat($old_schema);
+pass("Stub class KinoSearch::Schema passed by eat()");
+
+my $schema;
+SKIP: {
+ skip( "constructor bailouts cause leaks", 1 ) if $ENV{KINO_VALGRIND};
+
+ $schema = KinoSearch::Test::TestSchema->new;
+ eval { $schema->spec_field( name => 'foo', type => 'NotAType' ) };
+ Test::More::like( $@, qr/FieldType/, "bogus FieldType fails to load" );
+}
+
+$schema = KinoSearch::Test::TestSchema->new;
+my $type = $schema->fetch_type('content');
+$schema->spec_field( name => 'new_field', type => $type );
+my $got = grep { $_ eq 'new_field' } @{ $schema->all_fields };
+ok( $got, 'spec_field works' );
+
Added: incubator/lucy/trunk/import/perl/t/217-poly_lexicon.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/217-poly_lexicon.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/217-poly_lexicon.t (added)
+++ incubator/lucy/trunk/import/perl/t/217-poly_lexicon.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,106 @@
+use strict;
+use warnings;
+
+use lib 'buildlib';
+use KinoSearch::Test;
+
+use strict;
+use warnings;
+
+package MyArchitecture;
+use base qw( KinoSearch::Plan::Architecture );
+
+sub index_interval { confess("should be displaced via local() below") }
+
+package MySchema;
+use base qw( KinoSearch::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = KinoSearch::Plan::StringType->new( sortable => 1 );
+ $self->spec_field( name => 'content', type => $type );
+ return $self;
+}
+
+sub architecture { MyArchitecture->new }
+
+package main;
+use Test::More tests => 35;
+use File::Spec::Functions qw( catfile );
+
+my @docs = ( ( ("a") x 100 ), 'j', "Foo" );
+my @chars = ( 'a' .. 'z' );
+for ( 0 .. 100 ) {
+ my $content = '';
+ for my $num_chars ( 1 .. int( rand(10) + 1 ) ) {
+ $content .= @chars[ rand(@chars) ];
+ }
+ push @docs, "$content";
+}
+
+# Accumulate unique sorted terms.
+my @correct = ();
+for ( sort @docs ) {
+ next if ( @correct and $_ eq $correct[-1] );
+ push @correct, $_;
+}
+
+# Remember where 'j' exists in our 'correct' list.
+my $correct_term_num = 0;
+for (@correct) {
+ last if $correct[$correct_term_num] eq 'j';
+ $correct_term_num++;
+}
+
+for my $index_interval ( 1, 2, 3, 4, 7, 128, 1024 ) {
+
+ no warnings 'redefine';
+ local *MyArchitecture::index_interval = sub {$index_interval};
+
+ my $folder = KinoSearch::Store::RAMFolder->new;
+ my $schema = MySchema->new;
+
+ my @docs_copy = @docs;
+ while (@docs_copy) {
+ my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ );
+ my $docs_this_seg = int( rand(@docs_copy) );
+ $docs_this_seg = 10 if $docs_this_seg < 10;
+
+ for ( 0 .. 10 ) {
+ last unless @docs_copy;
+ my $tick = int( rand(@docs_copy) );
+ $indexer->add_doc(
+ { content => splice( @docs_copy, $tick, 1 ) } );
+ }
+ $indexer->commit;
+ }
+
+ my $reader = KinoSearch::Index::IndexReader->open( index => $folder, );
+
+ my $lexicon = $reader->obtain("KinoSearch::Index::LexiconReader")
+ ->lexicon( field => 'content' );
+ isa_ok( $lexicon, "KinoSearch::Index::PolyLexicon" );
+
+ $lexicon->next;
+ is( $lexicon->get_term, $correct[0],
+ "calling lexicon() without term returns Lexicon with iterator reset"
+ );
+ $lexicon->reset;
+
+ my @got;
+ while ( $lexicon->next ) {
+ push @got, $lexicon->get_term;
+ }
+
+ is_deeply( \@got, \@correct,
+ "correct order for random strings (interval: $index_interval)" );
+
+ $lexicon->seek('j');
+ is( $lexicon->get_term, 'j', "seek (interval: $index_interval)" );
+
+ $lexicon->seek(undef);
+ ok( !defined $lexicon->get_term, "seek to undef resets" );
+}
Added: incubator/lucy/trunk/import/perl/t/218-del_merging.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/218-del_merging.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/218-del_merging.t (added)
+++ incubator/lucy/trunk/import/perl/t/218-del_merging.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,105 @@
+use strict;
+use warnings;
+
+use KinoSearch::Test;
+
+package NoMergeSeg1Manager;
+use base qw( KinoSearch::Index::IndexManager );
+sub recycle {
+ my $seg_readers = shift->SUPER::recycle(@_);
+ @$seg_readers = grep { $_->get_seg_num != 1 } @$seg_readers;
+ return $seg_readers;
+}
+
+package DelSchema;
+use base 'KinoSearch::Plan::Schema';
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = KinoSearch::Plan::FullTextType->new(
+ analyzer => KinoSearch::Analysis::Tokenizer->new, );
+ $self->spec_field( name => 'foo', type => $type );
+ $self->spec_field( name => 'bar', type => $type );
+ return $self;
+}
+
+package main;
+
+use Test::More tests => 70;
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $schema = DelSchema->new;
+my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->add_doc( { foo => 'foo', bar => $_ } ) for qw( x y z );
+$indexer->commit;
+
+for my $iter ( 1 .. 10 ) {
+ is( search_doc('foo'), 3, "match all docs prior to deletion $iter" );
+ is( search_doc('x'), 1, "match doc to be deleted $iter" );
+
+ $indexer = KinoSearch::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->delete_by_term( field => 'bar', term => 'x' );
+ $indexer->optimize;
+ $indexer->commit;
+
+ is( search_doc('x'), 0, "deletion successful $iter" );
+ is( search_doc('foo'), 2, "match all docs after deletion $iter" );
+
+ $indexer = KinoSearch::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->add_doc( { foo => 'foo', bar => 'x' } );
+ $indexer->optimize;
+ $indexer->commit;
+}
+
+$folder = KinoSearch::Store::RAMFolder->new;
+$schema = DelSchema->new;
+$indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+my @dox = ( 'a' .. 'z' );
+$indexer->add_doc( { foo => 'foo', bar => $_ } ) for @dox;
+$indexer->commit;
+
+for ( 1 .. 10 ) {
+ $indexer = KinoSearch::Index::Indexer->new(
+ manager => NoMergeSeg1Manager->new,
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->delete_by_term( field => 'bar', term => 'y' );
+ $indexer->commit;
+ my @num_seg_1_bv_files = grep {/deletions-seg_1.bv/} @{ $folder->list_r };
+ is( scalar @num_seg_1_bv_files,
+ 1, "seg_1 deletions file carried forward" );
+
+ $indexer = KinoSearch::Index::Indexer->new(
+ manager => NoMergeSeg1Manager->new,
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->delete_by_term( field => 'bar', term => 'new' );
+ $indexer->add_doc( { foo => 'foo', bar => 'new' } );
+ $indexer->commit;
+ @num_seg_1_bv_files = grep {/deletions-seg_1.bv/} @{ $folder->list_r };
+ is( scalar @num_seg_1_bv_files,
+ 1, "seg_1 deletions file carried forward" );
+
+ is( search_doc('foo'), scalar @dox, "deletions applied correctly" );
+}
+
+sub search_doc {
+ my $query_string = shift;
+ my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+ my $hits = $searcher->hits( query => $query_string );
+ return $hits->total_hits;
+}
Added: incubator/lucy/trunk/import/perl/t/219-byte_buf_doc.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/219-byte_buf_doc.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/219-byte_buf_doc.t (added)
+++ incubator/lucy/trunk/import/perl/t/219-byte_buf_doc.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,98 @@
+use strict;
+use warnings;
+
+package MyArchitecture;
+use base qw( KinoSearch::Plan::Architecture );
+
+use KSx::Index::ByteBufDocWriter;
+use KSx::Index::ByteBufDocReader;
+
+sub register_doc_writer {
+ my ( $self, $seg_writer ) = @_;
+ my $doc_writer = KSx::Index::ByteBufDocWriter->new(
+ width => 1,
+ field => 'id',
+ schema => $seg_writer->get_schema,
+ snapshot => $seg_writer->get_snapshot,
+ segment => $seg_writer->get_segment,
+ polyreader => $seg_writer->get_polyreader,
+ );
+ $seg_writer->register(
+ api => "KinoSearch::Index::DocReader",
+ component => $doc_writer,
+ );
+ $seg_writer->add_writer($doc_writer);
+}
+
+sub register_doc_reader {
+ my ( $self, $seg_reader ) = @_;
+ my $doc_reader = KSx::Index::ByteBufDocReader->new(
+ width => 1,
+ schema => $seg_reader->get_schema,
+ folder => $seg_reader->get_folder,
+ segments => $seg_reader->get_segments,
+ seg_tick => $seg_reader->get_seg_tick,
+ snapshot => $seg_reader->get_snapshot,
+ );
+ $seg_reader->register(
+ api => 'KinoSearch::Index::DocReader',
+ component => $doc_reader,
+ );
+}
+
+package MySchema;
+use base qw( KinoSearch::Plan::Schema );
+
+sub architecture { MyArchitecture->new }
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $tokenizer = KinoSearch::Analysis::Tokenizer->new;
+ my $type = KinoSearch::Plan::FullTextType->new( analyzer => $tokenizer );
+ $self->spec_field( name => 'id', type => $type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 4;
+use KinoSearch::Test;
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $schema = MySchema->new;
+
+sub add_to_index {
+ my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ );
+ $indexer->add_doc( { id => $_ } ) for @_;
+ $indexer->commit;
+}
+
+add_to_index(qw( a b c ));
+
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+my $hits = $searcher->hits( query => 'b' );
+is( $hits->next, 'b', "single segment, single hit" );
+
+add_to_index(qw( d e f g h ));
+add_to_index(qw( i j k l m ));
+
+$searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'f' );
+is( $hits->next, 'f', "multiple segments, single hit" );
+
+my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'id', term => $_ ) for qw( b f l );
+$indexer->optimize;
+$indexer->commit;
+
+$searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'b' );
+is( $hits->next, undef, "doc deleted" );
+
+$hits = $searcher->hits( query => 'c' );
+is( $hits->next, 'c', "map around deleted doc" );
Added: incubator/lucy/trunk/import/perl/t/220-zlib_doc.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/220-zlib_doc.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/220-zlib_doc.t (added)
+++ incubator/lucy/trunk/import/perl/t/220-zlib_doc.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,119 @@
+use strict;
+use warnings;
+
+package MyArchitecture;
+use base qw( KinoSearch::Plan::Architecture );
+
+use KSx::Index::ZlibDocWriter;
+use KSx::Index::ZlibDocReader;
+
+sub register_doc_writer {
+ my ( $self, $seg_writer ) = @_;
+ my $doc_writer = KSx::Index::ZlibDocWriter->new(
+ schema => $seg_writer->get_schema,
+ snapshot => $seg_writer->get_snapshot,
+ segment => $seg_writer->get_segment,
+ polyreader => $seg_writer->get_polyreader,
+ );
+ $seg_writer->register(
+ api => "KinoSearch::Index::DocReader",
+ component => $doc_writer,
+ );
+ $seg_writer->add_writer($doc_writer);
+}
+
+sub register_doc_reader {
+ my ( $self, $seg_reader ) = @_;
+ my $doc_reader = KSx::Index::ZlibDocReader->new(
+ schema => $seg_reader->get_schema,
+ folder => $seg_reader->get_folder,
+ segments => $seg_reader->get_segments,
+ seg_tick => $seg_reader->get_seg_tick,
+ snapshot => $seg_reader->get_snapshot,
+ );
+ $seg_reader->register(
+ api => 'KinoSearch::Index::DocReader',
+ component => $doc_reader,
+ );
+}
+
+package MySchema;
+use base qw( KinoSearch::Plan::Schema );
+
+sub architecture { MyArchitecture->new }
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $tokenizer = KinoSearch::Analysis::Tokenizer->new;
+ my $main_type
+ = KinoSearch::Plan::FullTextType->new( analyzer => $tokenizer );
+ my $unstored_type = KinoSearch::Plan::FullTextType->new(
+ analyzer => $tokenizer,
+ stored => 0,
+ );
+ my $blob_type = KinoSearch::Plan::BlobType->new( stored => 1 );
+ $self->spec_field( name => 'content', type => $main_type );
+ $self->spec_field( name => 'smiley', type => $main_type );
+ $self->spec_field( name => 'unstored', type => $unstored_type );
+ $self->spec_field( name => 'binary', type => $blob_type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 7;
+use KinoSearch::Test;
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $schema = MySchema->new;
+
+my $smiley = "\x{263a}";
+my $binary = pack( 'b4', 1, 2, 3, 4 );
+
+sub add_to_index {
+ my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ );
+ for (@_) {
+ $indexer->add_doc(
+ { content => $_,
+ binary => $binary,
+ smiley => $smiley,
+ unstored => $_,
+ }
+ );
+ }
+ $indexer->commit;
+}
+
+add_to_index(qw( a b c ));
+
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+my $hits = $searcher->hits( query => 'b' );
+my $hit = $hits->next;
+is( $hit->{content}, 'b', "single segment, single hit" );
+is( $hit->{smiley}, $smiley, "utf8 preserved" );
+is( $hit->{binary}, $binary, "blob field binary preserved" );
+ok( !defined( $hit->{unstored} ), "unstored" );
+
+add_to_index(qw( d e f g h ));
+add_to_index(qw( i j k l m ));
+
+$searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'f' );
+is( $hits->next->{content}, 'f', "multiple segments, single hit" );
+
+my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'content', term => $_ ) for qw( b f l );
+$indexer->optimize;
+$indexer->commit;
+
+$searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'b' );
+is( $hits->next, undef, "doc deleted" );
+
+$hits = $searcher->hits( query => 'c' );
+is( $hits->next->{content}, 'c', "map around deleted doc" );
Added: incubator/lucy/trunk/import/perl/t/221-sort_writer.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/221-sort_writer.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/221-sort_writer.t (added)
+++ incubator/lucy/trunk/import/perl/t/221-sort_writer.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,159 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package NonMergingIndexManager;
+use base qw( KinoSearch::Index::IndexManager );
+
+sub recycle {
+ return KinoSearch::Object::VArray->new( capacity => 0 );
+}
+
+package SortSchema;
+use base qw( KinoSearch::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $fulltext_type = KinoSearch::Plan::FullTextType->new(
+ analyzer => KinoSearch::Analysis::Tokenizer->new,
+ sortable => 1,
+ );
+ my $string_type = KinoSearch::Plan::StringType->new( sortable => 1 );
+ my $unsortable = KinoSearch::Plan::StringType->new;
+ $self->spec_field( name => 'name', type => $fulltext_type );
+ $self->spec_field( name => 'speed', type => $string_type );
+ $self->spec_field( name => 'weight', type => $string_type );
+ $self->spec_field( name => 'home', type => $string_type );
+ $self->spec_field( name => 'cat', type => $string_type );
+ $self->spec_field( name => 'wheels', type => $string_type );
+ $self->spec_field( name => 'unused', type => $string_type );
+ $self->spec_field( name => 'nope', type => $unsortable );
+ return $self;
+}
+
+package main;
+use KinoSearch::Test;
+use Test::More tests => 57;
+
+# Force frequent flushes.
+KinoSearch::Index::SortWriter::set_default_mem_thresh(100);
+
+my $airplane = {
+ name => 'airplane',
+ speed => '0200',
+ weight => '8000',
+ home => 'air',
+ cat => 'vehicle',
+ wheels => 3,
+ nope => 'nyet',
+};
+my $bike = {
+ name => 'bike',
+ speed => '0015',
+ weight => '0025',
+ home => 'land',
+ cat => 'vehicle',
+ wheels => 2,
+};
+my $car = {
+ name => 'car',
+ speed => '0070',
+ weight => '3000',
+ home => 'land',
+ cat => 'vehicle',
+ wheels => 4,
+};
+my $dirigible = {
+ name => 'dirigible',
+ speed => '0040',
+ weight => '0000',
+ home => 'air',
+ cat => 'vehicle',
+ # no "wheels" field -- test NULL/undef
+};
+my $elephant = {
+ name => 'elephant',
+ speed => '0020',
+ weight => '6000',
+ home => 'land',
+ cat => 'vehicle',
+ # no "wheels" field -- test NULL/undef
+};
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $schema = SortSchema->new;
+my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+# Add vehicles.
+$indexer->add_doc($_) for ( $airplane, $bike, $car );
+
+$indexer->commit;
+
+my $polyreader = KinoSearch::Index::IndexReader->open( index => $folder );
+my $seg_reader = $polyreader->get_seg_readers->[0];
+my $sort_reader = $seg_reader->obtain("KinoSearch::Index::SortReader");
+my $doc_reader = $seg_reader->obtain("KinoSearch::Index::DocReader");
+my $segment = $seg_reader->get_segment;
+
+for my $field (qw( name speed weight home cat wheels )) {
+ my $field_num = $segment->field_num($field);
+ ok( $folder->exists("seg_1/sort-$field_num.ord"),
+ "sort files written for $field" );
+ my $sort_cache = $sort_reader->fetch_sort_cache($field);
+ for ( 1 .. $seg_reader->doc_max ) {
+ is( $sort_cache->value( ord => $sort_cache->ordinal($_) ),
+ $doc_reader->fetch( doc_id => $_ )->{$field},
+ "correct cached value doc $_ "
+ );
+ }
+}
+
+for my $field (qw( unused nope )) {
+ my $field_num = $segment->field_num($field);
+ ok( !$folder->exists("seg_1/sort-$field_num.ord"),
+ "no sort files written for $field" );
+}
+
+# Add a second segment.
+$indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ manager => NonMergingIndexManager->new,
+);
+$indexer->add_doc($dirigible);
+$indexer->commit;
+
+# Consolidate everything, to test merging.
+$indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'name', term => 'bike' );
+$indexer->add_doc($elephant);
+$indexer->optimize;
+$indexer->commit;
+
+my $num_old_seg_files = scalar grep {m/seg_[12]/} @{ $folder->list_r };
+is( $num_old_seg_files, 0, "all files from earlier segments zapped" );
+
+$polyreader = KinoSearch::Index::IndexReader->open( index => $folder );
+$seg_reader = $polyreader->get_seg_readers->[0];
+$sort_reader = $seg_reader->obtain("KinoSearch::Index::SortReader");
+$doc_reader = $seg_reader->obtain("KinoSearch::Index::DocReader");
+$segment = $seg_reader->get_segment;
+
+for my $field (qw( name speed weight home cat wheels )) {
+ my $field_num = $segment->field_num($field);
+ ok( $folder->exists("seg_3/sort-$field_num.ord"),
+ "sort files written for $field" );
+ my $sort_cache = $sort_reader->fetch_sort_cache($field);
+ for ( 1 .. $seg_reader->doc_max ) {
+ is( $sort_cache->value( ord => $sort_cache->ordinal($_) ),
+ $doc_reader->fetch( doc_id => $_ )->{$field},
+ "correct cached value field $field doc $_ "
+ );
+ }
+}
Added: incubator/lucy/trunk/import/perl/t/224-lex_reader.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/224-lex_reader.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/224-lex_reader.t (added)
+++ incubator/lucy/trunk/import/perl/t/224-lex_reader.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,27 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 3;
+use KinoSearch::Test::TestUtils qw( create_index );
+
+my $folder = create_index(
+ "What's he building in there?",
+ "What's he building in there?",
+ "We have a right to know."
+);
+my $polyreader = KinoSearch::Index::IndexReader->open( index => $folder );
+my $seg_reader = $polyreader->get_seg_readers->[0];
+my $lex_reader = $seg_reader->obtain("KinoSearch::Index::LexiconReader");
+
+my $lexicon = $lex_reader->lexicon( field => 'content', term => 'building' );
+isa_ok( $lexicon, 'KinoSearch::Index::Lexicon',
+ "lexicon returns a KinoSearch::Index::Lexicon" );
+my $tinfo = $lexicon->get_term_info;
+is( $tinfo->get_doc_freq, 2, "correct place in lexicon" );
+
+$lexicon = $lex_reader->lexicon( field => 'content' );
+$lexicon->next;
+is( $lexicon->get_term, 'We',
+ 'calling lexicon without a term returns Lexicon with iterator reset' );
+
Added: incubator/lucy/trunk/import/perl/t/233-background_merger.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/233-background_merger.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/233-background_merger.t (added)
+++ incubator/lucy/trunk/import/perl/t/233-background_merger.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,74 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package NoMergeManager;
+use base qw( KinoSearch::Index::IndexManager );
+sub recycle { [] }
+
+package main;
+use Test::More tests => 15;
+use KinoSearch::Test;
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $schema = KinoSearch::Test::TestSchema->new;
+
+for my $letter (qw( a b c )) {
+ my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ manager => NoMergeManager->new,
+ );
+ $indexer->add_doc( { content => $letter } );
+ $indexer->commit;
+}
+my $bg_merger = KinoSearch::Index::BackgroundMerger->new( index => $folder );
+
+my $indexer = KinoSearch::Index::Indexer->new( index => $folder );
+$indexer->add_doc( { content => 'd' } );
+$indexer->commit;
+
+is( count_segs($folder), 4,
+ "BackgroundMerger prevents Indexer from merging claimed segments" );
+
+$indexer = KinoSearch::Index::Indexer->new( index => $folder );
+$indexer->add_doc( { content => 'e' } );
+$indexer->delete_by_term( field => 'content', term => 'b' );
+$indexer->commit;
+is( count_segs($folder), 4, "Indexer may still merge unclaimed segments" );
+
+$bg_merger->commit;
+is( count_segs($folder), 3, "Background merge completes" );
+ok( $folder->exists("seg_7/deletions-seg_4.bv"),
+ "deletions carried forward" );
+
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+is( $searcher->hits( query => 'b' )->total_hits,
+ 0, "deleted term still deleted" );
+is( $searcher->hits( query => $_ )->total_hits, 1, "term $_ still present" )
+ for qw( a c d e );
+
+# Simulate failed background merge.
+$bg_merger = KinoSearch::Index::BackgroundMerger->new( index => $folder );
+$bg_merger->prepare_commit;
+undef $bg_merger;
+$folder->delete("merge.lock");
+die "test set up failed" unless $folder->exists("merge.json");
+
+$indexer = KinoSearch::Index::Indexer->new( index => $folder );
+$indexer->optimize;
+$indexer->commit;
+
+ok( !$folder->exists("merge.json"), "Cleaned up after failed bg merge" );
+
+$searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+is( $searcher->hits( query => 'b' )->total_hits,
+ 0, "deleted term still deleted after full optimize" );
+is( $searcher->hits( query => $_ )->total_hits,
+ 1, "term $_ still present after full optimize" )
+ for qw( a c d e );
+
+sub count_segs {
+ my $folder = shift;
+ return scalar grep {m/segmeta\.json/} @{ $folder->list_r };
+}
Added: incubator/lucy/trunk/import/perl/t/302-many_fields.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/302-many_fields.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/302-many_fields.t (added)
+++ incubator/lucy/trunk/import/perl/t/302-many_fields.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,46 @@
+use strict;
+use warnings;
+
+package MySchema;
+use base qw( KinoSearch::Plan::Schema );
+use KinoSearch::Analysis::Tokenizer;
+
+our %fields = ();
+
+package main;
+
+use Test::More tests => 10;
+use KinoSearch::Test;
+
+my $schema = MySchema->new;
+my $type = KinoSearch::Plan::FullTextType->new(
+ analyzer => KinoSearch::Analysis::Tokenizer->new, );
+
+for my $num_fields ( 1 .. 10 ) {
+ # Build an index with $num_fields fields, and the same content in each.
+ $schema->spec_field( name => "field$num_fields", type => $type );
+ my $folder = KinoSearch::Store::RAMFolder->new;
+ my $indexer = KinoSearch::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+
+ for my $content ( 'a' .. 'z', 'x x y' ) {
+ my %doc;
+ for ( 1 .. $num_fields ) {
+ $doc{"field$_"} = $content;
+ }
+ $indexer->add_doc( \%doc );
+ }
+ $indexer->commit;
+
+ # See if our search results match as expected.
+ my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+ my $hits = $searcher->hits(
+ query => 'x',
+ num_wanted => 100,
+ );
+ is( $hits->total_hits, 2,
+ "correct number of hits for $num_fields fields" );
+ my $top_hit = $hits->next;
+}
Added: incubator/lucy/trunk/import/perl/t/303-highlighter.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/303-highlighter.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/303-highlighter.t (added)
+++ incubator/lucy/trunk/import/perl/t/303-highlighter.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,393 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MySchema;
+use base qw( KinoSearch::Plan::Schema );
+use KinoSearch::Analysis::Tokenizer;
+
+sub new {
+ my $class = shift;
+ my $self = $class->SUPER::new(@_);
+ my $tokenizer = KinoSearch::Analysis::Tokenizer->new;
+ my $plain_type = KinoSearch::Plan::FullTextType->new(
+ analyzer => $tokenizer,
+ highlightable => 1,
+ );
+ my $dunked_type = KinoSearch::Plan::FullTextType->new(
+ analyzer => $tokenizer,
+ highlightable => 1,
+ boost => 0.1,
+ );
+ $self->spec_field( name => 'content', type => $plain_type );
+ $self->spec_field( name => 'alt', type => $dunked_type );
+ return $self;
+}
+
+package MyHighlighter;
+use base qw( KinoSearch::Highlight::Highlighter );
+
+sub encode {
+ my ( $self, $text ) = @_;
+ $text =~ s/blind/wise/;
+ return $text;
+}
+
+sub highlight {
+ my ( $self, $text ) = @_;
+ return "*$text*";
+}
+
+package main;
+
+use Test::More tests => 34;
+use KinoSearch::Test;
+
+binmode( STDOUT, ":utf8" );
+
+my $phi = "\x{03a6}";
+my $encoded_phi = "Φ";
+
+my $string = '1 2 3 4 5 ' x 20; # 200 characters
+$string .= "$phi a b c d x y z h i j k ";
+$string .= '6 7 8 9 0 ' x 20;
+my $with_quotes = '"I see," said the blind man.';
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => MySchema->new,
+);
+
+$indexer->add_doc( { content => $_ } ) for ( $string, $with_quotes );
+$indexer->add_doc(
+ { content => "x but not why or 2ee",
+ alt => $string . " and extra stuff so it scores lower",
+ }
+);
+$indexer->commit;
+
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+
+my $q = qq|"x y z" AND $phi|;
+my $hits = $searcher->hits( query => $q );
+my $hl = KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ excerpt_length => 3,
+);
+
+my $target = KinoSearch::Object::ViewCharBuf->_new("");
+
+my $field_val = make_cb("a $phi $phi b c");
+my $top = $hl->_find_best_fragment(
+ fragment => $target,
+ field_val => $field_val,
+ heat_map => make_heat_map( [ 2, 1, 1.0 ] ),
+);
+is( $target->to_perl, "$phi $phi b", "Find_Best_Fragment" );
+is( $top, 2, "correct offset returned by Find_Best_Fragment" );
+
+$field_val = make_cb("aa$phi");
+$top = $hl->_find_best_fragment(
+ fragment => $target,
+ field_val => $field_val,
+ heat_map => make_heat_map( [ 2, 1, 1.0 ] ),
+);
+is( $target->to_perl, $field_val->to_perl,
+ "Find_Best_Fragment returns whole field when field is short" );
+is( $top, 0, "correct offset" );
+
+$field_val = make_cb("aaaab$phi$phi");
+$top = $hl->_find_best_fragment(
+ fragment => $target,
+ field_val => $field_val,
+ heat_map => make_heat_map( [ 6, 2, 1.0 ] ),
+);
+is( $target->to_perl, "b$phi$phi",
+ "Find_Best_Fragment shifts left to deal with overrun" );
+is( $top, 4, "correct offset" );
+
+$field_val = make_cb( "a$phi" . "bcde" );
+$top = $hl->_find_best_fragment(
+ fragment => $target,
+ field_val => $field_val,
+ heat_map => make_heat_map( [ 0, 1, 1.0 ] ),
+);
+is( $target->to_perl,
+ "a$phi" . "bcd",
+ "Find_Best_Fragment start at field beginning"
+);
+is( $top, 0, "correct offset" );
+undef $target;
+
+$hl = KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ excerpt_length => 6,
+);
+
+$target = make_cb("");
+$field_val = "Ook. Urk. Ick. ";
+$top = $hl->_raw_excerpt(
+ field_val => $field_val,
+ fragment => "Ook. Urk.",
+ raw_excerpt => $target,
+ top => 0,
+ sentences => make_spans( [ 0, 4, 0 ], [ 6, 4, 0 ] ),
+ heat_map => make_heat_map( [ 0, length($field_val), 1.0 ] ),
+);
+is( $target->to_perl, "Ook.", "Raw_Excerpt at top" );
+is( $top, 0, "top still 0" );
+
+$target = make_cb("");
+$field_val = "Ook. Urk. Ick. ";
+$top = $hl->_raw_excerpt(
+ field_val => $field_val,
+ fragment => ". Urk. I",
+ raw_excerpt => $target,
+ top => 3,
+ sentences => make_spans( [ 6, 4, 0 ], [ 12, 4, 0 ] ),
+ heat_map => make_heat_map( [ 0, length($field_val), 1.0 ] ),
+);
+is( $target->to_perl, "Urk.", "Raw_Excerpt in middle, with 2 bounds" );
+is( $top, 6, "top in the middle modified by Raw_Excerpt" );
+
+$target = make_cb("");
+$field_val = "Ook urk ick i.";
+$top = $hl->_raw_excerpt(
+ field_val => $field_val,
+ fragment => "ick i.",
+ raw_excerpt => $target,
+ top => 8,
+ sentences => make_spans( [ 0, length($field_val), 0 ] ),
+ heat_map => make_heat_map( [ 0, length($field_val), 1.0 ] ),
+);
+is( $target->to_perl, "\x{2026} i.", "Ellipsis at top" );
+is( $top, 10, "top correct when leading ellipsis inserted" );
+
+$target = make_cb("");
+$field_val = "Urk. Iz no good.";
+$top = $hl->_raw_excerpt(
+ field_val => $field_val,
+ fragment => " Iz no go",
+ raw_excerpt => $target,
+ top => 4,
+ sentences => make_spans( [ 6, length($field_val) - 6, 0 ] ),
+ heat_map => make_heat_map( [ 0, length($field_val), 1.0 ] ),
+);
+is( $target->to_perl, "Iz no\x{2026}", "Ellipsis at end" );
+is( $top, 6, "top trimmed" );
+
+$hl = KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ excerpt_length => 3,
+);
+
+$target = make_cb("");
+$hl->_highlight_excerpt(
+ raw_excerpt => 'a b c',
+ spans => make_spans( [ 2, 1 ] ),
+ top => 0,
+ highlighted => $target,
+);
+is( $target->to_perl, "a <strong>b</strong> c", "basic Highlight_Excerpt" );
+
+$target = make_cb("");
+$hl->_highlight_excerpt(
+ raw_excerpt => "$phi $phi $phi",
+ spans => make_spans( [ 2, 1, 1.0 ] ),
+ top => 0,
+ highlighted => $target,
+);
+like(
+ $target->to_perl,
+ qr#$encoded_phi <strong>$encoded_phi</strong> $encoded_phi#i,
+ "encode invoked by Highlight_Excerpt"
+);
+
+$target = make_cb("");
+$hl->_highlight_excerpt(
+ raw_excerpt => "$phi $phi $phi",
+ spans => make_spans( [ 3, 1, 1.0 ] ),
+ top => 1,
+ highlighted => $target,
+);
+like(
+ $target->to_perl,
+ qr#^$encoded_phi <strong>$encoded_phi</strong> $encoded_phi$#i,
+ "Highlight_Excerpt pays attention to offset"
+);
+
+$hl = KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+);
+
+my $hit = $hits->next;
+my $excerpt = $hl->create_excerpt($hit);
+like( $excerpt, qr/$encoded_phi.*?z/i,
+ "excerpt contains all relevant terms" );
+like( $excerpt, qr#<strong>x y z</strong>#, "highlighter tagged the phrase" );
+like(
+ $excerpt,
+ qr#<strong>$encoded_phi</strong>#i,
+ "highlighter tagged the single term"
+);
+
+$hl->set_pre_tag("\e[1m");
+$hl->set_post_tag("\e[0m");
+like(
+ $hl->create_excerpt($hit),
+ qr#\e\[1m$encoded_phi\e\[0m#i, "set_pre_tag and set_post_tag",
+);
+
+like( $hl->create_excerpt( $hits->next() ),
+ qr/x/,
+ "excerpt field with partial hit doesn't cause highlighter freakout" );
+
+$hits = $searcher->hits( query => $q = 'x "x y z" AND b' );
+$hl = KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+);
+$excerpt = $hl->create_excerpt( $hits->next() );
+$excerpt =~ s#</?strong>##g;
+like( $excerpt, qr/x y z/,
+ "query with same word in both phrase and term doesn't cause freakout" );
+
+$hits = $searcher->hits( query => $q = 'blind' );
+like(
+ KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ )->create_excerpt( $hits->next() ),
+ qr/quot/,
+ "HTML entity encoded properly"
+);
+
+$hits = $searcher->hits( query => $q = 'why' );
+unlike(
+ KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ )->create_excerpt( $hits->next() ),
+ qr/\.\.\./,
+ "no ellipsis for short excerpt"
+);
+
+my $term_query = KinoSearch::Search::TermQuery->new(
+ field => 'content',
+ term => 'x',
+);
+$hits = $searcher->hits( query => $term_query );
+$hit = $hits->next();
+like(
+ KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $term_query,
+ field => 'content',
+ )->create_excerpt($hit),
+ qr/strong/,
+ "specify field highlights correct field..."
+);
+unlike(
+ KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $term_query,
+ field => 'alt',
+ )->create_excerpt($hit),
+ qr/strong/,
+ "... but not another field"
+);
+
+my $sentence_text = 'This is a sentence. ' x 15;
+$hl = KinoSearch::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+);
+my $sentences = $hl->find_sentences(
+ text => $sentence_text,
+ offset => 101,
+ length => 50,
+);
+is_deeply(
+ spans_to_arg_array($sentences),
+ [ [ 120, 19, 0 ], [ 140, 19, 0 ] ],
+ 'find_sentences with explicit args'
+);
+
+$sentences = $hl->find_sentences(
+ text => $sentence_text,
+ offset => 101,
+ length => 4,
+);
+is_deeply( spans_to_arg_array($sentences),
+ [], 'find_sentences with explicit args, finding nothing' );
+
+my @expected;
+for my $i ( 0 .. 14 ) {
+ push @expected, [ $i * 20, 19, 0 ];
+}
+$sentences = $hl->find_sentences( text => $sentence_text );
+is_deeply( spans_to_arg_array($sentences),
+ \@expected, 'find_sentences with default offset and length' );
+
+$sentences = $hl->find_sentences( text => ' Foo' );
+is_deeply(
+ spans_to_arg_array($sentences),
+ [ [ 1, 3, 0 ] ],
+ "Skip leading whitespace but get first sentence"
+);
+
+$hl = MyHighlighter->new(
+ searcher => $searcher,
+ query => "blind",
+ field => 'content',
+);
+$hits = $searcher->hits( query => 'blind' );
+$hit = $hits->next;
+like( $hl->create_excerpt($hit),
+ qr/\*wise\*/, "override both Encode() and Highlight()" );
+
+sub make_cb {
+ return KinoSearch::Object::CharBuf->new(shift);
+}
+
+sub make_heat_map {
+ return KinoSearch::Highlight::HeatMap->new( spans => make_spans(@_) );
+}
+
+sub make_span {
+ return KinoSearch::Search::Span->new(
+ offset => $_[0],
+ length => $_[1],
+ weight => $_[2],
+ );
+}
+
+sub make_spans {
+ my $spans = KinoSearch::Object::VArray->new( capacity => scalar @_ );
+ for my $span_spec (@_) {
+ $spans->push( make_span( @{$span_spec}[ 0 .. 2 ] ) );
+ }
+ return $spans;
+}
+
+sub spans_to_arg_array {
+ my $spans = shift;
+ my @out;
+ for (@$spans) {
+ push @out, [ $_->get_offset, $_->get_length, $_->get_weight ];
+ }
+ return \@out;
+}
Added: incubator/lucy/trunk/import/perl/t/304-verify_utf8.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/304-verify_utf8.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/304-verify_utf8.t (added)
+++ incubator/lucy/trunk/import/perl/t/304-verify_utf8.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,102 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MySchema;
+use base qw( KinoSearch::Plan::Schema );
+use KinoSearch::Analysis::Tokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $analyzer = KinoSearch::Analysis::Tokenizer->new( pattern => '\S+' );
+ my $type = KinoSearch::Plan::FullTextType->new( analyzer => $analyzer, );
+ $self->spec_field( name => 'content', type => $type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 14;
+use KinoSearch::Test;
+use KinoSearch::Test::TestUtils qw( utf8_test_strings );
+
+my ( $smiley, $not_a_smiley, $frowny ) = utf8_test_strings();
+
+my $turd = pack( 'C*', 254, 254 );
+my $polished_turd = $turd;
+utf8::upgrade($polished_turd);
+
+is( $turd, $polished_turd, "verify encoding acrobatics" );
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $schema = MySchema->new;
+my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+$indexer->add_doc( { content => $smiley } );
+$indexer->add_doc( { content => $not_a_smiley } );
+$indexer->add_doc( { content => $turd } );
+$indexer->commit;
+
+my $qparser = KinoSearch::Search::QueryParser->new( schema => MySchema->new );
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+
+my $hits = $searcher->hits( query => $qparser->parse($smiley) );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content},
+ $smiley, "Indexer and QueryParser handle UTF-8 source correctly" );
+
+$hits = $searcher->hits( query => $qparser->parse($frowny) );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content}, $frowny, "Indexer upgrades non-UTF-8 correctly" );
+
+$hits = $searcher->hits( query => $qparser->parse($not_a_smiley) );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content},
+ $not_a_smiley, "QueryParser upgrades non-UTF-8 correctly" );
+
+my $term_query = KinoSearch::Search::TermQuery->new(
+ field => 'content',
+ term => $not_a_smiley,
+);
+$hits = $searcher->hits( query => $term_query );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content},
+ $not_a_smiley, "TermQuery upgrades non-UTF-8 correctly" );
+
+$term_query = KinoSearch::Search::TermQuery->new(
+ field => 'content',
+ term => $smiley,
+);
+
+$hits = $searcher->hits( query => $term_query );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content}, $smiley, "TermQuery handles UTF-8 correctly" );
+
+undef $indexer;
+$indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'content', term => $smiley );
+$indexer->commit;
+$searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+
+$hits = $searcher->hits( query => $smiley );
+is( $hits->total_hits, 0, "delete_by_term handles UTF-8 correctly" );
+
+$hits = $searcher->hits( query => $frowny );
+is( $hits->total_hits, 1, "delete_by_term handles UTF-8 correctly" );
+
+undef $indexer;
+$indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'content', term => $not_a_smiley );
+$indexer->commit;
+$searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+
+$hits = $searcher->hits( query => $frowny );
+is( $hits->total_hits, 0, "delete_by_term upgrades non-UTF-8 correctly" );
Added: incubator/lucy/trunk/import/perl/t/305-indexer.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/305-indexer.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/305-indexer.t (added)
+++ incubator/lucy/trunk/import/perl/t/305-indexer.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,73 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 4;
+use KinoSearch::Test;
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $schema = KinoSearch::Test::TestSchema->new;
+
+my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+$indexer->add_doc( { content => 'foo' } );
+undef $indexer;
+
+$indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->add_doc( { content => 'foo' } );
+pass("Indexer ignores garbage from interrupted session");
+
+SKIP: {
+ skip( "Known leak, though might be fixable", 2 ) if $ENV{KINO_VALGRIND};
+ eval {
+ my $manager
+ = KinoSearch::Index::IndexManager->new( host => 'somebody_else' );
+ my $inv = KinoSearch::Index::Indexer->new(
+ manager => $manager,
+ index => $folder,
+ schema => $schema,
+ );
+ };
+ like( $@, qr/write.lock/, "failed to get lock with competing host" );
+ isa_ok( $@, "KinoSearch::Store::LockErr", "Indexer throws a LockErr" );
+}
+
+my $pid = 12345678;
+do {
+ # Fake a write lock.
+ $folder->delete("locks/write.lock") or die "Couldn't delete 'write.lock'";
+ my $outstream = $folder->open_out('locks/write.lock')
+ or die KinoSearch->error;
+ while ( kill( 0, $pid ) ) {
+ $pid++;
+ }
+ $outstream->print(
+ qq|
+ {
+ "host": "somebody_else",
+ "pid": $pid,
+ "name": "write"
+ }|
+ );
+ $outstream->close;
+
+ eval {
+ my $manager
+ = KinoSearch::Index::IndexManager->new( host => 'somebody_else' );
+ my $inv = KinoSearch::Index::Indexer->new(
+ manager => $manager,
+ schema => $schema,
+ index => $folder,
+ );
+ };
+
+ # Mitigate (though not eliminate) race condition false failure.
+} while ( kill( 0, $pid ) );
+
+ok( !$@, "clobbered lock from same host with inactive pid" );
Added: incubator/lucy/trunk/import/perl/t/306-dynamic_schema.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/306-dynamic_schema.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/306-dynamic_schema.t (added)
+++ incubator/lucy/trunk/import/perl/t/306-dynamic_schema.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,84 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 9;
+use KinoSearch::Test;
+
+my $schema = KinoSearch::Test::TestSchema->new;
+my $type = $schema->fetch_type('content');
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $indexer = KinoSearch::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+);
+
+my %one_field = ( content => 'x 1' );
+my %two_fields = ( content => 'x x 2', a => 'a' );
+my %three_fields = ( content => 'x x x 3', a => 'a', b => 'b' );
+my %four_fields = ( content => 'x x x 3', a => 'a', b => 'b', c => 'c', );
+my %foo_doc = ( content => 'foo', foo => 'foo' );
+my %five_fields = (
+ content => 'x x x x x 5',
+ a => 'a',
+ b => 'b',
+ c => 'c',
+ foo => 'stuff'
+);
+
+$indexer->add_doc( \%one_field );
+$schema->spec_field( name => 'a', type => $type );
+$indexer->add_doc( \%two_fields );
+pass('Add a field in the middle of indexing');
+
+$schema->spec_field( name => 'a', type => $type );
+pass('Add same field again');
+
+$schema->spec_field( name => 'b', type => $type );
+$indexer->add_doc( \%three_fields );
+pass('Add another field');
+$indexer->commit;
+
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+my $hits = $searcher->hits( query => 'x', num_wanted => 100 );
+is( $hits->total_hits, 3,
+ "disparate docs successfully indexed and retrieved" );
+my $top_hit = $hits->next;
+is_deeply( $top_hit->get_fields, \%three_fields,
+ "all fields stored successfully" );
+
+my $schema2 = KinoSearch::Test::TestSchema->new;
+my $folder2 = KinoSearch::Store::RAMFolder->new;
+$schema2->spec_field( name => 'foo', type => $type );
+my $indexer2 = KinoSearch::Index::Indexer->new(
+ schema => $schema2,
+ index => $folder2,
+);
+$indexer2->add_doc( \%foo_doc );
+$indexer2->commit;
+
+undef $indexer;
+$indexer = KinoSearch::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+);
+
+$schema->spec_field( name => 'c', type => $type );
+$indexer->add_doc( \%four_fields );
+
+$indexer->add_index($folder2);
+$indexer->add_doc( \%five_fields );
+pass('successfully absorbed new field def during add_index');
+$indexer->commit;
+
+$searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'stuff', num_wanted => 100 );
+is( $hits->total_hits, 1,
+ "successfully aborbed unknown field during add_index" );
+$top_hit = $hits->next;
+delete $top_hit->{score};
+is_deeply( $top_hit->get_fields, \%five_fields,
+ "all fields stored successfully" );
+
+$hits = $searcher->hits( query => 'x', num_wanted => 100 );
+is( $hits->total_hits, 5, "indexes successfully merged" );
Added: incubator/lucy/trunk/import/perl/t/308-simple.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/308-simple.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/308-simple.t (added)
+++ incubator/lucy/trunk/import/perl/t/308-simple.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,72 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 8;
+use KSx::Simple;
+use KinoSearch::Test::TestUtils qw( init_test_index_loc );
+
+my $test_index_loc = init_test_index_loc();
+
+my $index = KSx::Simple->new(
+ language => 'en',
+ path => $test_index_loc,
+);
+
+$index->add_doc( { food => 'creamed corn' } );
+is( $index->search( query => 'creamed' ), 1, "search warks right after add" );
+
+$index->add_doc( { food => 'creamed spinach' } );
+is( $index->search( query => 'creamed' ), 2, "search returns total hits" );
+
+$index->add_doc( { food => 'creamed broccoli' } );
+undef $index;
+$index = KSx::Simple->new(
+ language => 'en',
+ path => $test_index_loc,
+);
+is( $index->search( query => 'cream' ), 3, "commit upon destroy" );
+
+while ( my $hit = $index->next ) {
+ like( $hit->{food}, qr/cream/, 'next' );
+}
+
+$index->add_doc( { band => 'Cream' } );
+is( $index->search( query => 'cream' ),
+ 4, "search uses correct PolyAnalyzer" );
+
+SKIP: {
+ skip( "fork on Windows not supported by KS", 1 ) if $^O =~ /mswin/i;
+
+ # We need another one:
+ my $test_index_loc = init_test_index_loc();
+
+ # Fork a process that will create an index without explicitly finishing
+ # it, and then exit, with the Simple object still in existence at
+ # global destruction time.
+ my $pid = fork();
+ if ( $pid == 0 ) { # child
+ our # This *has* to be 'our' for the test to work
+ $index = KSx::Simple->new(
+ language => 'en',
+ path => $test_index_loc,
+ );
+
+ $index->add_doc( { food => 'creamed corn' } );
+ exit;
+ }
+ else {
+ waitpid( $pid, 0 );
+ }
+
+ my $index = KSx::Simple->new(
+ language => 'en',
+ path => $test_index_loc,
+ );
+
+ ok eval { # This should die if the index wasn't finished.
+ $index->search( query => 'creamed' );
+ 1;
+ }, 'Simple finishes indexing during END block (apparently)';
+
+}
Added: incubator/lucy/trunk/import/perl/t/309-span.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/309-span.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/309-span.t (added)
+++ incubator/lucy/trunk/import/perl/t/309-span.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,27 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 6;
+
+use KinoSearch::Test;
+use KinoSearch::Search::Span;
+
+my $span = KinoSearch::Search::Span->new(
+ offset => 2,
+ length => 3,
+ weight => 7,
+);
+
+is( $span->get_offset, 2, "get_offset" );
+is( $span->get_length, 3, "get_length" );
+is( $span->get_weight, 7, "get_weight" );
+
+$span->set_offset(10);
+$span->set_length(1);
+$span->set_weight(4);
+
+is( $span->get_offset, 10, "set_offset" );
+is( $span->get_length, 1, "set_length" );
+is( $span->get_weight, 4, "set_weight" );
+
Added: incubator/lucy/trunk/import/perl/t/310-heat_map.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/310-heat_map.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/310-heat_map.t (added)
+++ incubator/lucy/trunk/import/perl/t/310-heat_map.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,128 @@
+use strict;
+use warnings;
+
+use Test::More tests => 13;
+use KinoSearch::Test;
+
+my $heat_map = KinoSearch::Highlight::HeatMap->new( spans => [], );
+
+my $big_boost = $heat_map->calc_proximity_boost(
+ span1 => make_span( 0, 10, 1.0 ),
+ span2 => make_span( 10, 10, 1.0 )
+);
+my $equally_big_boost = $heat_map->calc_proximity_boost(
+ span1 => make_span( 0, 10, 1.0 ),
+ span2 => make_span( 5, 4, 1.0 )
+);
+my $smaller_boost = $heat_map->calc_proximity_boost(
+ span1 => make_span( 0, 10, 1.0 ),
+ span2 => make_span( 100, 10, 1.0 )
+);
+my $zero_boost = $heat_map->calc_proximity_boost(
+ span1 => make_span( 0, 10, 1.0 ),
+ span2 => make_span( 150, 10, 1.0 )
+);
+is( $big_boost, $equally_big_boost,
+ "overlapping and abutting produce the same proximity boost" );
+cmp_ok( $big_boost, '>', $smaller_boost, "closer is better" );
+is( $zero_boost, 0, "distance outside of window yields no prox boost" );
+
+my $spans = make_spans( [ 10, 10, 1.0 ], [ 16, 14, 2.0 ] );
+my $flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 6, 1.0 ], [ 16, 4, 3.0 ], [ 20, 10, 2.0 ] ],
+ "flatten two overlapping spans"
+);
+my $boosts = $heat_map->generate_proximity_boosts($spans);
+is_deeply(
+ spans_to_arg_array($boosts),
+ [ [ 10, 20, 3.0 ] ],
+ "prox boosts for overlap"
+);
+
+$spans = make_spans( [ 10, 10, 1.0 ], [ 16, 14, 2.0 ], [ 50, 1, 1.0 ] );
+$flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 6, 1.0 ], [ 16, 4, 3.0 ], [ 20, 10, 2.0 ], [ 50, 1, 1.0 ] ],
+ "flatten two overlapping spans, leave hole, then third span"
+);
+$boosts = $heat_map->generate_proximity_boosts($spans);
+is( scalar @$boosts,
+ 2 + 1, "boosts generated for each unique pair, since all were in range" );
+
+$spans = make_spans( [ 10, 10, 1.0 ], [ 14, 4, 4.0 ], [ 16, 14, 2.0 ] );
+$flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 4, 1.0 ],
+ [ 14, 2, 5.0 ],
+ [ 16, 2, 7.0 ],
+ [ 18, 2, 3.0 ],
+ [ 20, 10, 2.0 ]
+ ],
+ "flatten three overlapping spans"
+);
+$boosts = $heat_map->generate_proximity_boosts($spans);
+is( scalar @$boosts,
+ 2 + 1, "boosts generated for each unique pair, since all were in range" );
+
+$spans = make_spans(
+ [ 10, 10, 1.0 ],
+ [ 16, 14, 4.0 ],
+ [ 16, 14, 2.0 ],
+ [ 30, 10, 10.0 ]
+);
+$flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 6, 1.0 ], [ 16, 4, 7.0 ], [ 20, 10, 6.0 ], [ 30, 10, 10.0 ] ],
+ "flatten 4 spans, middle two have identical range"
+);
+$boosts = $heat_map->generate_proximity_boosts($spans);
+is( scalar @$boosts,
+ 3 + 2 + 1,
+ "boosts generated for each unique pair, since all were in range"
+);
+
+$spans = make_spans(
+ [ 10, 10, 1.0 ],
+ [ 16, 4, 4.0 ],
+ [ 16, 14, 2.0 ],
+ [ 230, 10, 10.0 ]
+);
+$flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 6, 1.0 ], [ 16, 4, 7.0 ], [ 20, 10, 2.0 ], [ 230, 10, 10.0 ] ],
+ "flatten 4 spans, middle two have identical starts but different ends"
+);
+$boosts = $heat_map->generate_proximity_boosts($spans);
+is( scalar @$boosts, 2 + 1, "boosts not generated for out of range span" );
+
+sub make_span {
+ return KinoSearch::Search::Span->new(
+ offset => $_[0],
+ length => $_[1],
+ weight => $_[2],
+ );
+}
+
+sub make_spans {
+ my @spans;
+ for my $arg_ref (@_) {
+ push @spans, make_span( @{$arg_ref}[ 0 .. 2 ] );
+ }
+ return \@spans;
+}
+
+sub spans_to_arg_array {
+ my $spans = shift;
+ my @out;
+ for (@$spans) {
+ push @out, [ $_->get_offset, $_->get_length, $_->get_weight ];
+ }
+ return \@out;
+}
+
Added: incubator/lucy/trunk/import/perl/t/400-match_posting.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/400-match_posting.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/400-match_posting.t (added)
+++ incubator/lucy/trunk/import/perl/t/400-match_posting.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,81 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MatchOnlySim;
+use base qw( KinoSearch::Index::Similarity );
+
+sub make_posting {
+ my $self = shift;
+ return KinoSearch::Index::Posting::MatchPosting->new(
+ similarity => $self );
+}
+
+package MatchSchema::MatchOnly;
+use base qw( KinoSearch::Plan::FullTextType );
+use KinoSearch::Index::Posting::MatchPosting;
+
+sub make_similarity { MatchOnlySim->new }
+
+package MatchSchema;
+use base qw( KinoSearch::Plan::Schema );
+use KinoSearch::Analysis::Tokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = MatchSchema::MatchOnly->new(
+ analyzer => KinoSearch::Analysis::Tokenizer->new );
+ $self->spec_field( name => 'content', type => $type );
+ return $self;
+}
+
+package main;
+
+use KinoSearch::Test::TestUtils qw( get_uscon_docs );
+use Test::More tests => 6;
+
+my $uscon_docs = get_uscon_docs();
+my $match_folder = make_index( MatchSchema->new, $uscon_docs );
+my $score_folder
+ = make_index( KinoSearch::Test::TestSchema->new, $uscon_docs );
+
+my $match_searcher
+ = KinoSearch::Search::IndexSearcher->new( index => $match_folder );
+my $score_searcher
+ = KinoSearch::Search::IndexSearcher->new( index => $score_folder );
+
+for (qw( land of the free )) {
+ my $match_got = hit_ids_array( $match_searcher, $_ );
+ my $score_got = hit_ids_array( $score_searcher, $_ );
+ is_deeply( $match_got, $score_got, "same hits for '$_'" );
+}
+
+my $qstring = '"the legislature"';
+my $should_have_hits = hit_ids_array( $score_searcher, $qstring );
+my $should_be_empty = hit_ids_array( $match_searcher, $qstring );
+ok( scalar @$should_have_hits, "successfully scored phrase $qstring" );
+ok( !scalar @$should_be_empty, "no hits matched for phrase $qstring" );
+
+sub make_index {
+ my ( $schema, $docs ) = @_;
+ my $folder = KinoSearch::Store::RAMFolder->new;
+ my $indexer = KinoSearch::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->add_doc( { content => $_->{bodytext} } ) for values %$docs;
+ $indexer->commit;
+ return $folder;
+}
+
+sub hit_ids_array {
+ my ( $searcher, $query_string ) = @_;
+ my $query = $searcher->glean_query($query_string);
+
+ my $bit_vec = KinoSearch::Object::BitVector->new(
+ capacity => $searcher->doc_max + 1 );
+ my $bit_collector = KinoSearch::Search::Collector::BitCollector->new(
+ bit_vector => $bit_vec, );
+ $searcher->collect( query => $query, collector => $bit_collector );
+ return $bit_vec->to_array->to_arrayref;
+}
Added: incubator/lucy/trunk/import/perl/t/501-termquery.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/501-termquery.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/501-termquery.t (added)
+++ incubator/lucy/trunk/import/perl/t/501-termquery.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,49 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 12;
+use Storable qw( freeze thaw );
+use KinoSearch::Test;
+use KinoSearch::Test::TestUtils qw( create_index );
+
+my $folder = create_index( 'a', 'b', 'c c c d', 'c d', 'd' .. 'z', );
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+
+my $term_query
+ = KinoSearch::Search::TermQuery->new( field => 'content', term => 'c' );
+is( $term_query->to_string, "content:c", "to_string" );
+
+my $hits = $searcher->hits( query => $term_query );
+is( $hits->total_hits, 2, "correct number of hits returned" );
+
+my $hit = $hits->next;
+is( $hit->{content}, 'c c c d', "most relevant doc is highest" );
+
+$hit = $hits->next;
+is( $hit->{content}, 'c d', "second most relevant" );
+
+my $frozen = freeze($term_query);
+my $thawed = thaw($frozen);
+is( $thawed->get_field, 'content', "field survives freeze/thaw" );
+is( $thawed->get_term, 'c', "term survives freeze/thaw" );
+is( $thawed->get_boost, $term_query->get_boost,
+ "boost survives freeze/thaw" );
+ok( $thawed->equals($term_query), "equals" );
+$thawed->set_boost(10);
+ok( !$thawed->equals($term_query), "!equals (boost)" );
+my $different_term = KinoSearch::Search::TermQuery->new(
+ field => 'content',
+ term => 'd'
+);
+my $different_field = KinoSearch::Search::TermQuery->new(
+ field => 'title',
+ term => 'c'
+);
+ok( !$term_query->equals($different_term), "!equals (term)" );
+ok( !$term_query->equals($different_field), "!equals (field)" );
+
+my $term_compiler = $term_query->make_compiler( searcher => $searcher );
+$frozen = freeze($term_compiler);
+$thawed = thaw($frozen);
+ok( $term_compiler->equals($thawed), "freeze/thaw compiler" );
Added: incubator/lucy/trunk/import/perl/t/502-phrasequery.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/502-phrasequery.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/502-phrasequery.t (added)
+++ incubator/lucy/trunk/import/perl/t/502-phrasequery.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,69 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 9;
+use Storable qw( freeze thaw );
+use KinoSearch::Test;
+use KinoSearch::Test::TestUtils qw( create_index );
+
+my $best_match = 'x a b c d a b c d';
+
+my @docs = (
+ 1 .. 20,
+ 'a b c a b c a b c d',
+ 'a b c d x x a',
+ 'a c b d', 'a x x x b x x x c x x x x x x d x',
+ $best_match, 'a' .. 'z',
+);
+
+my $folder = create_index(@docs);
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+
+my $phrase_query = KinoSearch::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [],
+);
+is( $phrase_query->to_string, 'content:""', "empty PhraseQuery to_string" );
+$phrase_query = KinoSearch::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [qw( a b c d )],
+);
+is( $phrase_query->to_string, 'content:"a b c d"', "to_string" );
+
+my $hits = $searcher->hits( query => $phrase_query );
+is( $hits->total_hits, 3, "correct number of hits" );
+my $first_hit = $hits->next;
+is( $first_hit->{content}, $best_match, 'best match appears first' );
+
+my $second_hit = $hits->next;
+ok( $first_hit->get_score > $second_hit->get_score,
+ "best match scores higher: "
+ . $first_hit->get_score . " > "
+ . $second_hit->get_score
+);
+
+$phrase_query = KinoSearch::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [qw( c a )],
+);
+$hits = $searcher->hits( query => $phrase_query );
+is( $hits->total_hits, 1, 'avoid underflow when subtracting offset' );
+
+# "a b c"
+$phrase_query = KinoSearch::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [qw( a b c )],
+);
+$hits = $searcher->hits( query => $phrase_query );
+is( $hits->total_hits, 3, 'offset starting from zero' );
+
+my $frozen = freeze($phrase_query);
+my $thawed = thaw($frozen);
+$hits = $searcher->hits( query => $thawed );
+is( $hits->total_hits, 3, 'freeze/thaw' );
+
+my $phrase_compiler = $phrase_query->make_compiler( searcher => $searcher );
+$frozen = freeze($phrase_compiler);
+$thawed = thaw($frozen);
+ok( $phrase_compiler->equals($thawed), "freeze/thaw compiler" );
Added: incubator/lucy/trunk/import/perl/t/504-similarity.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/504-similarity.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/504-similarity.t (added)
+++ incubator/lucy/trunk/import/perl/t/504-similarity.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,106 @@
+use strict;
+use warnings;
+
+package MySchema::LongTextField;
+use base qw( KinoSearch::Plan::FullTextType );
+use KSx::Index::LongFieldSim;
+
+sub make_similarity { KSx::Index::LongFieldSim->new }
+
+package MySchema;
+use base qw( KinoSearch::Plan::Schema );
+use KinoSearch::Analysis::Tokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $analyzer = KinoSearch::Analysis::Tokenizer->new;
+ my $plain_type
+ = KinoSearch::Plan::FullTextType->new( analyzer => $analyzer, );
+ my $long_field_type
+ = MySchema::LongTextField->new( analyzer => $analyzer, );
+ $self->spec_field( name => 'title', type => $plain_type );
+ $self->spec_field( name => 'body', type => $long_field_type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 9;
+use KinoSearch::Test;
+use bytes;
+no bytes;
+
+my $sim = KinoSearch::Index::Similarity->new;
+my $evil_twin = $sim->load( $sim->dump );
+ok( $sim->equals($evil_twin), "Dump/Load" );
+
+cmp_ok( $sim->tf(10) - $sim->tf(9), '<', 1, "TF is damped" );
+
+my $rare_idf = $sim->idf( doc_freq => 3, total_docs => 100 );
+my $common_idf = $sim->idf( doc_freq => 50, total_docs => 100 );
+cmp_ok( $rare_idf, '>', $common_idf, 'Rarer terms have higher IDF' );
+
+my $less_coordinated = $sim->coord( overlap => 2, max_overlap => 5 );
+my $more_coordinated = $sim->coord( overlap => 3, max_overlap => 5 );
+cmp_ok( $less_coordinated, '<', $more_coordinated,
+ "greater overlap means bigger coord bonus" );
+
+my @bytes = ( 100, 110, 120, 130, 140 );
+my @floats = ( 0.015625, 0.09375, 0.5, 3.0, 16.0 );
+my @transformed = map { $sim->decode_norm($_) } @bytes;
+is_deeply( \@floats, \@transformed,
+ "decode_norm more or less matches Java Lucene behavior" );
+
+@bytes = 0 .. 255;
+@floats = map { $sim->decode_norm($_) } @bytes;
+@transformed = map { $sim->encode_norm($_) } @floats;
+is_deeply( \@transformed, \@bytes,
+ "encode_norm and decode_norm are complementary" );
+
+my $norm_decoder = $sim->get_norm_decoder;
+@transformed = ();
+for ( 0 .. 255 ) {
+ push @transformed,
+ unpack( 'f', bytes::substr( $norm_decoder, $_ * 4, 4 ) );
+}
+is_deeply( \@transformed, \@floats,
+ "using the norm_decoder produces desired results" );
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => MySchema->new,
+);
+
+my %source_docs = (
+ 'spam' => 'spam spam',
+ 'not spam' => 'not spam not even close to spam no spam here',
+);
+while ( my ( $title, $body ) = each %source_docs ) {
+ $indexer->add_doc(
+ { title => $title,
+ body => $body,
+ }
+ );
+}
+$indexer->commit;
+undef $indexer;
+
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+
+my $hits = $searcher->hits(
+ query => KinoSearch::Search::TermQuery->new(
+ field => 'title',
+ term => 'spam',
+ )
+);
+is( $hits->next->{'title'},
+ 'spam', "Default Similarity biased towards short fields" );
+
+$hits = $searcher->hits(
+ query => KinoSearch::Search::TermQuery->new(
+ field => 'body',
+ term => 'spam',
+ )
+);
+is( $hits->next->{'title'},
+ 'not spam', "LongFieldSim cancels short-field bias" );
Added: incubator/lucy/trunk/import/perl/t/505-hit_queue.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/505-hit_queue.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/505-hit_queue.t (added)
+++ incubator/lucy/trunk/import/perl/t/505-hit_queue.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,198 @@
+use strict;
+use warnings;
+
+use Test::More tests => 36;
+use KinoSearch::Test;
+use List::Util qw( shuffle );
+
+my $hit_q = KinoSearch::Search::HitQueue->new( wanted => 10 );
+
+my @docs_and_scores = (
+ [ 0, 1.0 ],
+ [ 5, 0.1 ],
+ [ 10, 0.1 ],
+ [ 1000, 0.9 ],
+ [ 2000, 1.0 ],
+ [ 3000, 1.0 ],
+);
+
+my @match_docs = map {
+ KinoSearch::Search::MatchDoc->new(
+ doc_id => $_->[0],
+ score => $_->[1],
+ )
+} @docs_and_scores;
+
+my @correct_order = sort {
+ $b->get_score <=> $a->get_score
+ or $a->get_doc_id <=> $b->get_doc_id
+} @match_docs;
+my @correct_docs = map { $_->get_doc_id } @correct_order;
+my @correct_scores = map { $_->get_score } @correct_order;
+
+$hit_q->insert($_) for @match_docs;
+my $got = $hit_q->pop_all;
+
+my @scores = map { $_->get_score } @$got;
+is_deeply( \@scores, \@correct_scores, "rank by scores first" );
+
+my @doc_ids = map { $_->get_doc_id } @$got;
+is_deeply( \@doc_ids, \@correct_docs, "rank by doc_id after score" );
+
+my $schema = KinoSearch::Plan::Schema->new;
+my $type = KinoSearch::Plan::StringType->new( sortable => 1 );
+$schema->spec_field( name => 'letter', type => $type );
+$schema->spec_field( name => 'number', type => $type );
+$schema->spec_field( name => 'id', type => $type );
+
+my $folder = KinoSearch::Store::RAMFolder->new;
+my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+my @letters = 'a' .. 'z';
+my @numbers = 1 .. 5;
+my @docs;
+for my $id ( 1 .. 100 ) {
+ my $doc = {
+ letter => $letters[ rand @letters ],
+ number => $numbers[ rand @numbers ],
+ id => $id,
+ };
+ push @docs, $doc;
+ $indexer->add_doc($doc);
+}
+$indexer->commit;
+my $seg_reader = KinoSearch::Index::IndexReader->open( index => $folder );
+
+my $by_letter = KinoSearch::Search::SortSpec->new(
+ rules => [
+ KinoSearch::Search::SortRule->new( field => 'letter' ),
+ KinoSearch::Search::SortRule->new( type => 'doc_id' ),
+ ]
+);
+$hit_q = KinoSearch::Search::HitQueue->new(
+ wanted => 100,
+ schema => $schema,
+ sort_spec => $by_letter,
+);
+for my $doc_id ( shuffle( 1 .. 100 ) ) {
+ my $match_doc = KinoSearch::Search::MatchDoc->new(
+ doc_id => $doc_id,
+ score => 1.0,
+ values => [ $docs[ $doc_id - 1 ]{letter} ],
+ );
+ $hit_q->insert($match_doc);
+}
+my @wanted = map { $_->{id} }
+ sort { $a->{letter} cmp $b->{letter} || $a->{id} <=> $b->{id} } @docs;
+my @got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+is_deeply( \@got, \@wanted, "sort by letter then doc id" );
+
+my $by_num_then_letter = KinoSearch::Search::SortSpec->new(
+ rules => [
+ KinoSearch::Search::SortRule->new( field => 'number' ),
+ KinoSearch::Search::SortRule->new( field => 'letter', reverse => 1 ),
+ KinoSearch::Search::SortRule->new( type => 'doc_id' ),
+ ]
+);
+$hit_q = KinoSearch::Search::HitQueue->new(
+ wanted => 100,
+ schema => $schema,
+ sort_spec => $by_num_then_letter,
+);
+for my $doc_id ( shuffle( 1 .. 100 ) ) {
+ my $doc = $docs[ $doc_id - 1 ];
+ my $match_doc = KinoSearch::Search::MatchDoc->new(
+ doc_id => $doc_id,
+ score => 1.0,
+ values => [ $doc->{number}, $doc->{letter} ],
+ );
+ $hit_q->insert($match_doc);
+}
+@wanted = map { $_->{id} }
+ sort {
+ $a->{number} <=> $b->{number}
+ || $b->{letter} cmp $a->{letter}
+ || $a->{id} <=> $b->{id}
+ } @docs;
+@got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+is_deeply( \@got, \@wanted, "sort by num then letter reversed then doc id" );
+
+my $by_num_then_score = KinoSearch::Search::SortSpec->new(
+ rules => [
+ KinoSearch::Search::SortRule->new( field => 'number' ),
+ KinoSearch::Search::SortRule->new( type => 'score' ),
+ KinoSearch::Search::SortRule->new( type => 'doc_id' ),
+ ]
+);
+$hit_q = KinoSearch::Search::HitQueue->new(
+ wanted => 100,
+ schema => $schema,
+ sort_spec => $by_num_then_score,
+);
+
+for my $doc_id ( shuffle( 1 .. 100 ) ) {
+ my $match_doc = KinoSearch::Search::MatchDoc->new(
+ doc_id => $doc_id,
+ score => $doc_id,
+ values => [ $docs[ $doc_id - 1 ]{number} ],
+ );
+ $hit_q->insert($match_doc);
+}
+@wanted = map { $_->{id} }
+ sort { $a->{number} <=> $b->{number} || $b->{id} <=> $a->{id} } @docs;
+@got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+is_deeply( \@got, \@wanted, "sort by num then score then doc id" );
+
+$hit_q = KinoSearch::Search::HitQueue->new(
+ wanted => 100,
+ schema => $schema,
+ sort_spec => $by_num_then_score,
+);
+
+for my $doc_id ( shuffle( 1 .. 100 ) ) {
+ my $fields = $docs[ $doc_id - 1 ];
+ my $values = KinoSearch::Object::VArray->new( capacity => 1 );
+ $values->push( KinoSearch::Object::CharBuf->new( $fields->{number} ) );
+ my $match_doc = KinoSearch::Search::MatchDoc->new(
+ doc_id => $doc_id,
+ score => $doc_id,
+ values => $values,
+ );
+ $hit_q->insert($match_doc);
+}
+@wanted = map { $_->{id} }
+ sort { $a->{number} <=> $b->{number} || $b->{id} <=> $a->{id} } @docs;
+@got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+is_deeply( \@got, \@wanted, "sort by value when no reader set" );
+
+@docs_and_scores = ();
+for ( 1 .. 30 ) {
+ push @docs_and_scores, [ int( rand(10000) ) + 1, rand(10) ];
+}
+@docs_and_scores = sort { $b->[1] <=> $a->[1] } @docs_and_scores;
+@doc_ids = map { $_->[0] } @docs_and_scores;
+
+@match_docs = map {
+ KinoSearch::Search::MatchDoc->new(
+ doc_id => $_->[0],
+ score => $_->[1],
+ )
+} sort { $a->[0] <=> $b->[0] } @docs_and_scores;
+
+for my $size ( 0 .. $#match_docs ) {
+ $hit_q = KinoSearch::Search::HitQueue->new( wanted => $size, );
+ $hit_q->insert($_) for @match_docs;
+
+ if ($size) {
+ @wanted = @doc_ids[ 0 .. $size - 1 ];
+ }
+ else {
+ @wanted = ();
+ }
+ @got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+ is_deeply( \@got, \@wanted, "random docs and scores, size $size" );
+}
+
Added: incubator/lucy/trunk/import/perl/t/506-collector.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/506-collector.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/506-collector.t (added)
+++ incubator/lucy/trunk/import/perl/t/506-collector.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,55 @@
+use strict;
+use warnings;
+
+use Test::More tests => 5;
+use KinoSearch::Test;
+use KSx::Search::MockScorer;
+
+my @docs = ( 1, 5, 10, 1000 );
+my @scores = ( 2, 0, 0, 1 );
+
+my $collector
+ = KinoSearch::Search::Collector::SortCollector->new( wanted => 3 );
+test_collect($collector);
+
+my @got = map { $_->get_score } @{ $collector->pop_match_docs };
+is_deeply( \@got, [ 2, 1, 0 ], "collect into HitQueue" );
+
+$collector = KinoSearch::Search::Collector::SortCollector->new( wanted => 0 );
+test_collect($collector);
+is( $collector->get_total_hits, 4,
+ "get_total_hits is accurate when no hits are requested" );
+my $match_docs = $collector->pop_match_docs;
+is( scalar @$match_docs, 0, "no hits wanted, so no hits returned" );
+
+my $bit_vec = KinoSearch::Object::BitVector->new;
+$collector = KinoSearch::Search::Collector::BitCollector->new(
+ bit_vector => $bit_vec );
+test_collect($collector);
+is_deeply(
+ $bit_vec->to_arrayref,
+ [ 1, 5, 10, 1000 ],
+ "BitCollector collects the right doc nums"
+);
+
+$bit_vec = KinoSearch::Object::BitVector->new;
+my $inner_coll = KinoSearch::Search::Collector::BitCollector->new(
+ bit_vector => $bit_vec );
+my $offset_coll = KinoSearch::Search::Collector::OffsetCollector->new(
+ collector => $inner_coll,
+ offset => 10,
+);
+test_collect($offset_coll);
+is_deeply( $bit_vec->to_arrayref, [ 11, 15, 20, 1010 ], "Offset collector" );
+
+sub test_collect {
+ my $collector = shift;
+ my $matcher = KSx::Search::MockScorer->new(
+ doc_ids => \@docs,
+ scores => \@scores,
+ );
+ $collector->set_matcher($matcher);
+ while ( my $doc_id = $matcher->next ) {
+ $collector->collect($doc_id);
+ }
+}
Added: incubator/lucy/trunk/import/perl/t/507-filter.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/507-filter.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/507-filter.t (added)
+++ incubator/lucy/trunk/import/perl/t/507-filter.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,115 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 24;
+use Storable qw( nfreeze thaw );
+use KinoSearch::Test;
+use KinoSearch::Test::TestUtils qw( create_index );
+use KSx::Search::Filter;
+
+my $query_parser = KinoSearch::Search::QueryParser->new(
+ schema => KinoSearch::Test::TestSchema->new );
+
+## Set up main objects.
+my ( $filter_1, $filter_2 );
+{
+ my $folder_1 = create_index( 'a x', 'b x', 'c x', 'a y', 'b y', 'c y' );
+ my $folder_2 = create_index( 'a w', 'b w', 'c w', 'a z', 'b z', 'c z' );
+
+ my $searcher_1
+ = KinoSearch::Search::IndexSearcher->new( index => $folder_1 );
+ my $searcher_2
+ = KinoSearch::Search::IndexSearcher->new( index => $folder_2 );
+
+ my $reader_1 = $searcher_1->get_reader->get_seg_readers->[0];
+ my $reader_2 = $searcher_2->get_reader->get_seg_readers->[0];
+
+ my $only_a_query = KinoSearch::Search::TermQuery->new(
+ field => 'content',
+ term => 'a',
+ );
+ $filter_1 = KSx::Search::Filter->new( query => $only_a_query );
+ $filter_2 = KSx::Search::Filter->new( query => $only_a_query );
+
+ ## Test index 1, filter 1.
+ my $hits = $searcher_1->hits( query => filt_query( $filter_1, 'x y z' ) );
+ is( $hits->total_hits, 2, 'filtering a query works' );
+
+ ## Test index 2, filter 2.
+ $hits = $searcher_2->hits( query => filt_query( $filter_2, 'x y z' ) );
+ is( $hits->total_hits, 1, 'filtering a query works' );
+
+ ## Compare 1-1 to 2-2.
+ my $cached_bits_1 = $filter_1->_bits($reader_1);
+ my $cached_bits_2 = $filter_2->_bits($reader_2);
+ ok( !$cached_bits_1->equals($cached_bits_2),
+ 'cached bits are unique (1-1 != 2-2)'
+ );
+
+ ## Test copy of index 1, filter 1.
+ $hits = $searcher_1->hits( query => filt_query( $filter_1, 'w y z' ) );
+ my $bits = $filter_1->_bits($reader_1);
+ is( $hits->total_hits, 1, 'filtering a query works' );
+ ok( $cached_bits_1->equals($bits),
+ 'cached bits are cached (1-1 == 1-1)' );
+ ok( !$cached_bits_2->equals($bits),
+ 'cached bits are unique (2-2 != 1-1)'
+ );
+
+ ## Test index 1, filter 2.
+ $hits = $searcher_1->hits( query => filt_query( $filter_2, 'w y z' ) );
+ my $cached_bits_3 = $bits = $filter_2->_bits($reader_1);
+ is( $hits->total_hits, 1, 'filtering a query works' );
+ ok( !$cached_bits_1->equals($bits),
+ 'cached bits are unique (1-1 != 1-2)'
+ );
+ ok( !$cached_bits_2->equals($bits),
+ 'cached bits are unique (2-2 != 1-2)'
+ );
+
+ ## Test copy of index 2, filter 2.
+ $hits = $searcher_2->hits( query => filt_query( $filter_2, 'x y z' ) );
+ $bits = $filter_2->_bits($reader_2);
+ is( $hits->total_hits, 1, 'filtering a query works' );
+ ok( !$cached_bits_1->equals($bits),
+ 'cached bits are unique (1-1 != 2-2)'
+ );
+ ok( $cached_bits_2->equals($bits),
+ 'cached bits are cached (2-2 != 2-2)' );
+ ok( !$cached_bits_3->equals($bits),
+ 'cached bits are unique (1-2 != 2-2)'
+ );
+
+ ## Test copy of index 1, filter 2.
+ $hits = $searcher_1->hits( query => filt_query( $filter_2, 'x y z' ) );
+ $bits = $filter_2->_bits($reader_1);
+ is( $hits->total_hits, 2, 'filtering a query works' );
+ ok( !$cached_bits_1->equals($bits),
+ 'cached bits are unique (1-1 != 1-2)'
+ );
+ ok( !$cached_bits_2->equals($bits),
+ 'cached bits are unique (2-2 != 1-2)'
+ );
+ ok( $cached_bits_3->equals($bits),
+ 'cached bits are cached (1-2 != 1-2)' );
+
+ is( $filter_1->_cached_count, 1, 'cache count check correct' );
+ is( $filter_2->_cached_count, 2, 'cache count check correct' );
+}
+
+sub filt_query {
+ my ( $filter, $query_string ) = @_;
+ return KinoSearch::Search::ANDQuery->new(
+ children => [ $filter, $query_parser->parse($query_string) ], );
+}
+
+# Readers should be automatically undef'd, refcnt == 0.
+is( $filter_1->_cached_count, 0, 'cache count check correct' );
+is( $filter_2->_cached_count, 0, 'cache count check correct' );
+
+ok( $filter_1->equals($filter_2), 'equals' );
+my $frozen = nfreeze($filter_1);
+my $thawed = thaw($frozen);
+ok( $thawed->equals($filter_1), 'freeze/thaw' );
+is( $filter_1->to_string, "Filter(content:a)", "to_string" );
Added: incubator/lucy/trunk/import/perl/t/508-hits.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/508-hits.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/508-hits.t (added)
+++ incubator/lucy/trunk/import/perl/t/508-hits.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,50 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 9;
+use KinoSearch::Test;
+use KinoSearch::Test::TestUtils qw( create_index );
+
+my @docs = ( 'a b', 'a a b', 'a a a b', 'x' );
+my $folder = create_index(@docs);
+my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+
+my $hits = $searcher->hits(
+ query => 'a',
+ offset => 0,
+ num_wanted => 1,
+);
+is( $hits->total_hits, 3, "total_hits" );
+my $hit = $hits->next;
+cmp_ok( $hit->get_score, '>', 0.0, "score field added" );
+is( $hits->next, undef, "hits exhausted" );
+
+$hits->next;
+is( $hits->next, undef, "hits exhausted" );
+
+my @retrieved;
+@retrieved = ();
+$hits = $searcher->hits(
+ query => 'a',
+ offset => 0,
+ num_wanted => 100,
+);
+is( $hits->total_hits, 3, "total_hits still correct" );
+while ( my $hit = $hits->next ) {
+ push @retrieved, $hit->{content};
+}
+is_deeply( \@retrieved, [ @docs[ 2, 1, 0 ] ], "correct content via next()" );
+
+@retrieved = ();
+$hits = $searcher->hits(
+ query => 'a',
+ offset => 1,
+ num_wanted => 100,
+);
+is( $hits->total_hits, 3, "total_hits correct with offset" );
+while ( my $hit = $hits->next ) {
+ push @retrieved, $hit->{content};
+}
+is( scalar @retrieved, 2, "number retrieved with offset" );
+is_deeply( \@retrieved, [ @docs[ 1, 0 ] ], "correct content with offset" );
Added: incubator/lucy/trunk/import/perl/t/509-poly_searcher.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/509-poly_searcher.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/509-poly_searcher.t (added)
+++ incubator/lucy/trunk/import/perl/t/509-poly_searcher.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,40 @@
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 8;
+use KinoSearch::Test;
+use KinoSearch::Test::TestUtils qw( create_index );
+
+my $folder_a = create_index( 'x a', 'x b', 'x c' );
+my $folder_b = create_index( 'y b', 'y c', 'y d' );
+my $searcher_a = KinoSearch::Search::IndexSearcher->new( index => $folder_a );
+my $searcher_b = KinoSearch::Search::IndexSearcher->new( index => $folder_b );
+
+my $poly_searcher = KinoSearch::Search::PolySearcher->new(
+ schema => KinoSearch::Test::TestSchema->new,
+ searchers => [ $searcher_a, $searcher_b ],
+);
+
+is( $poly_searcher->doc_freq( field => 'content', term => 'b' ),
+ 2, 'doc_freq' );
+is( $poly_searcher->doc_max, 6, 'doc_max' );
+is( $poly_searcher->fetch_doc( doc_id => 1 )->{content}, 'x a', "fetch_doc" );
+isa_ok( $poly_searcher->fetch_doc_vec(1), 'KinoSearch::Index::DocVector' );
+
+my $hits = $poly_searcher->hits( query => 'a' );
+is( $hits->total_hits, 1, "Find hit in first searcher" );
+
+$hits = $poly_searcher->hits( query => 'd' );
+is( $hits->total_hits, 1, "Find hit in second searcher" );
+
+$hits = $poly_searcher->hits( query => 'c' );
+is( $hits->total_hits, 2, "Find hits in both searchers" );
+
+my $bit_vec = KinoSearch::Object::BitVector->new(
+ capacity => $poly_searcher->doc_max );
+my $bitcoll = KinoSearch::Search::Collector::BitCollector->new(
+ bit_vector => $bit_vec );
+my $query = $poly_searcher->glean_query('b');
+$poly_searcher->collect( query => $query, collector => $bitcoll );
+is_deeply( $bit_vec->to_arrayref, [ 2, 4 ], "collect" );
Added: incubator/lucy/trunk/import/perl/t/510-remote_search.t
URL: http://svn.apache.org/viewvc/incubator/lucy/trunk/import/perl/t/510-remote_search.t?rev=1022422&view=auto
==============================================================================
--- incubator/lucy/trunk/import/perl/t/510-remote_search.t (added)
+++ incubator/lucy/trunk/import/perl/t/510-remote_search.t Thu Oct 14 08:48:04 2010
@@ -0,0 +1,172 @@
+use strict;
+use warnings;
+
+use Test::More;
+use Time::HiRes qw( sleep );
+use IO::Socket::INET;
+
+my $PORT_NUM = 7890;
+BEGIN {
+ if ( $^O =~ /mswin/i ) {
+ plan( 'skip_all', "fork on Windows not supported by KS" );
+ }
+ elsif ( $ENV{KINO_VALGRIND} ) {
+ plan( 'skip_all', "time outs cause probs under valgrind" );
+ }
+}
+
+package SortSchema;
+use base qw( KinoSearch::Plan::Schema );
+use KinoSearch::Analysis::Tokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $plain_type = KinoSearch::Plan::FullTextType->new(
+ analyzer => KinoSearch::Analysis::Tokenizer->new );
+ my $string_type = KinoSearch::Plan::StringType->new( sortable => 1 );
+ $self->spec_field( name => 'content', type => $plain_type );
+ $self->spec_field( name => 'number', type => $string_type );
+ return $self;
+}
+
+package main;
+
+use KinoSearch::Test;
+use KSx::Remote::SearchServer;
+use KSx::Remote::SearchClient;
+
+my $kid;
+$kid = fork;
+if ($kid) {
+ sleep .25; # allow time for the server to set up the socket
+ die "Failed fork: $!" unless defined $kid;
+}
+else {
+ my $folder = KinoSearch::Store::RAMFolder->new;
+ my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder,
+ schema => SortSchema->new,
+ );
+ my $number = 5;
+ for (qw( a b c )) {
+ $indexer->add_doc( { content => "x $_", number => $number } );
+ $number -= 2;
+ }
+ $indexer->commit;
+
+ my $searcher = KinoSearch::Search::IndexSearcher->new( index => $folder );
+ my $server = KSx::Remote::SearchServer->new(
+ port => $PORT_NUM,
+ searcher => $searcher,
+ password => 'foo',
+ );
+ $server->serve;
+ exit(0);
+}
+
+my $test_client_sock = IO::Socket::INET->new(
+ PeerAddr => "localhost:$PORT_NUM",
+ Proto => 'tcp',
+);
+if ($test_client_sock) {
+ plan( tests => 10 );
+ undef $test_client_sock;
+}
+else {
+ plan( 'skip_all', "Can't get a socket: $!" );
+}
+
+my $searchclient = KSx::Remote::SearchClient->new(
+ schema => SortSchema->new,
+ peer_address => "localhost:$PORT_NUM",
+ password => 'foo',
+);
+
+is( $searchclient->doc_freq( field => 'content', term => 'x' ),
+ 3, "doc_freq" );
+is( $searchclient->doc_max, 3, "doc_max" );
+isa_ok( $searchclient->fetch_doc( doc_id => 1 ),
+ "KinoSearch::Document::HitDoc", "fetch_doc" );
+isa_ok(
+ $searchclient->fetch_doc_vec(1),
+ "KinoSearch::Index::DocVector",
+ "fetch_doc_vec"
+);
+
+my $hits = $searchclient->hits( query => 'x' );
+is( $hits->total_hits, 3, "retrieved hits from search server" );
+
+$hits = $searchclient->hits( query => 'a' );
+is( $hits->total_hits, 1, "retrieved hit from search server" );
+
+my $folder_b = KinoSearch::Store::RAMFolder->new;
+my $number = 6;
+for (qw( a b c )) {
+ my $indexer = KinoSearch::Index::Indexer->new(
+ index => $folder_b,
+ schema => SortSchema->new,
+ );
+ $indexer->add_doc( { content => "y $_", number => $number } );
+ $number -= 2;
+ $indexer->add_doc( { content => 'blah blah blah' } ) for 1 .. 3;
+ $indexer->commit;
+}
+
+my $searcher_b
+ = KinoSearch::Search::IndexSearcher->new( index => $folder_b, );
+is( ref( $searcher_b->get_reader ), 'KinoSearch::Index::PolyReader', );
+
+my $poly_searcher = KinoSearch::Search::PolySearcher->new(
+ schema => SortSchema->new,
+ searchers => [ $searcher_b, $searchclient ],
+);
+
+$hits = $poly_searcher->hits( query => 'b' );
+is( $hits->total_hits, 2, "retrieved hits from PolySearcher" );
+
+my %results;
+$results{ $hits->next()->{content} } = 1;
+$results{ $hits->next()->{content} } = 1;
+my %expected = ( 'x b' => 1, 'y b' => 1, );
+
+is_deeply( \%results, \%expected, "docs fetched from both local and remote" );
+
+my $sort_spec = KinoSearch::Search::SortSpec->new(
+ rules => [
+ KinoSearch::Search::SortRule->new( field => 'number' ),
+ KinoSearch::Search::SortRule->new( type => 'doc_id' ),
+ ],
+);
+$hits = $poly_searcher->hits(
+ query => 'b',
+ sort_spec => $sort_spec,
+);
+my @got;
+
+while ( my $hit = $hits->next ) {
+ push @got, $hit->{content};
+}
+$sort_spec = KinoSearch::Search::SortSpec->new(
+ rules => [
+ KinoSearch::Search::SortRule->new( field => 'number', reverse => 1 ),
+ KinoSearch::Search::SortRule->new( type => 'doc_id' ),
+ ],
+);
+$hits = $poly_searcher->hits(
+ query => 'b',
+ sort_spec => $sort_spec,
+);
+my @reversed;
+while ( my $hit = $hits->next ) {
+ push @reversed, $hit->{content};
+}
+is_deeply(
+ \@got,
+ [ reverse @reversed ],
+ "Sort combination of remote and local"
+);
+
+END {
+ $searchclient->terminate if defined $searchclient;
+ kill( TERM => $kid ) if $kid;
+}