You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ko...@apache.org on 2012/01/17 17:43:23 UTC
svn commit: r1232470 - in /lucene/dev/trunk/lucene/contrib: ./
highlighter/src/java/org/apache/lucene/search/vectorhighlight/
highlighter/src/test/org/apache/lucene/search/vectorhighlight/
Author: koji
Date: Tue Jan 17 16:43:23 2012
New Revision: 1232470
URL: http://svn.apache.org/viewvc?rev=1232470&view=rev
Log:
LUCENE-3698: FVH adds a multi value separator to the end of the highlighted text
Modified:
lucene/dev/trunk/lucene/contrib/CHANGES.txt
lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java
lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
Modified: lucene/dev/trunk/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/CHANGES.txt?rev=1232470&r1=1232469&r2=1232470&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/contrib/CHANGES.txt Tue Jan 17 16:43:23 2012
@@ -105,6 +105,11 @@ Changes in backwards compatibility polic
* LUCENE-3626: The internal implementation classes in PKIndexSplitter
and MultiPassIndexSplitter were made private as they now work
per segment. (Uwe Schindler)
+
+Changes in Runtime Behavior
+
+ * LUCENE-3698: FastVectorHighlighter no longer adds a multi value separator
+ to the end of the highlighted text. (Shay Banon via Koji Sekiguchi)
New Features
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java?rev=1232470&r1=1232469&r2=1232470&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Tue Jan 17 16:43:23 2012
@@ -166,7 +166,12 @@ public abstract class BaseFragmentsBuild
buffer.append( values[index[0]++].stringValue() );
buffer.append( getMultiValuedSeparator() );
}
- int eo = buffer.length() < endOffset ? buffer.length() : boundaryScanner.findEndOffset( buffer, endOffset );
+ int bufferLength = buffer.length();
+ // we added the multi value char to the last buffer, ignore it
+ if (values[index[0] - 1].fieldType().tokenized()) {
+ bufferLength--;
+ }
+ int eo = bufferLength < endOffset ? bufferLength : boundaryScanner.findEndOffset( buffer, endOffset );
modifiedStartOffset[0] = boundaryScanner.findStartOffset( buffer, startOffset );
return buffer.substring( modifiedStartOffset[0], eo );
}
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java?rev=1232470&r1=1232469&r2=1232470&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java Tue Jan 17 16:43:23 2012
@@ -35,7 +35,7 @@ public class ScoreOrderFragmentsBuilderT
String[] f = sofb.createFragments( reader, 0, F, ffl, 3 );
assertEquals( 3, f.length );
// check score order
- assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[0] );
+ assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
assertEquals( "<b>a</b> b b b b b b b b b b", f[2] );
}
Modified: lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java?rev=1232470&r1=1232469&r2=1232470&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java (original)
+++ lucene/dev/trunk/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Tue Jan 17 16:43:23 2012
@@ -37,11 +37,11 @@ public class SimpleFragmentsBuilderTest
public void test1TermIndex() throws Exception {
FieldFragList ffl = ffl(new TermQuery(new Term(F, "a")), "a" );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
- assertEquals( "<b>a</b> ", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( "<b>a</b>", sfb.createFragment( reader, 0, F, ffl ) );
// change tags
sfb = new SimpleFragmentsBuilder( new String[]{ "[" }, new String[]{ "]" } );
- assertEquals( "[a] ", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( "[a]", sfb.createFragment( reader, 0, F, ffl ) );
}
public void test2Frags() throws Exception {
@@ -51,7 +51,7 @@ public class SimpleFragmentsBuilderTest
// 3 snippets requested, but should be 2
assertEquals( 2, f.length );
assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
- assertEquals( "b b <b>a</b> b <b>a</b> b ", f[1] );
+ assertEquals( "b b <b>a</b> b <b>a</b> b", f[1] );
}
public void test3Frags() throws Exception {
@@ -65,7 +65,7 @@ public class SimpleFragmentsBuilderTest
assertEquals( 3, f.length );
assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
- assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[2] );
+ assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[2] );
}
public void testTagsAndEncoder() throws Exception {
@@ -73,7 +73,7 @@ public class SimpleFragmentsBuilderTest
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] preTags = { "[" };
String[] postTags = { "]" };
- assertEquals( "<h1> [a] </h1> ",
+ assertEquals( "<h1> [a] </h1>",
sfb.createFragment( reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder() ) );
}
@@ -94,7 +94,7 @@ public class SimpleFragmentsBuilderTest
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
- assertEquals( "a b c <b>d</b> e ", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( "a b c <b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
}
public void test1PhraseLongMV() throws Exception {
@@ -119,7 +119,7 @@ public class SimpleFragmentsBuilderTest
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
- assertEquals( "processing <b>speed</b>, the ", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( "processing <b>speed</b>, the", sfb.createFragment( reader, 0, F, ffl ) );
}
public void testUnstoredField() throws Exception {
@@ -160,7 +160,7 @@ public class SimpleFragmentsBuilderTest
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.setMultiValuedSeparator( '/' );
- assertEquals( "abc/<b>defg</b>/hijkl/", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( "abc/<b>defg</b>/hijkl", sfb.createFragment( reader, 0, F, ffl ) );
}
public void testMVSeparator() throws Exception {
@@ -173,6 +173,6 @@ public class SimpleFragmentsBuilderTest
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.setMultiValuedSeparator( '/' );
- assertEquals( " b c//<b>d</b> e/", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( " b c//<b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
}
}