You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ko...@apache.org on 2012/01/17 17:52:06 UTC
svn commit: r1232473 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/contrib/ lucene/contrib/analyzers/ lucene/contrib/analyzers/common/
lucene/contrib/analyzers/kuromoji/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighli...
Author: koji
Date: Tue Jan 17 16:52:05 2012
New Revision: 1232473
URL: http://svn.apache.org/viewvc?rev=1232473&view=rev
Log:
LUCENE-3698: FVH adds a multi value separator to the end of the highlighted text
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/analyzers/kuromoji/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java
lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
lucene/dev/branches/branch_3x/solr/ (props changed)
lucene/dev/branches/branch_3x/solr/core/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt?rev=1232473&r1=1232472&r2=1232473&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt Tue Jan 17 16:52:05 2012
@@ -10,6 +10,11 @@ Changes in backwards compatibility polic
* LUCENE-3626: The internal implementation classes in PKIndexSplitter
and MultiPassIndexSplitter were made private as they now work
per segment. (Uwe Schindler)
+
+Changes in Runtime Behavior
+
+ * LUCENE-3698: FastVectorHighlighter no longer adds a multi value separator
+ to the end of the highlighted text. (Shay Banon via Koji Sekiguchi)
New Features
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java?rev=1232473&r1=1232472&r2=1232473&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Tue Jan 17 16:52:05 2012
@@ -183,7 +183,12 @@ public abstract class BaseFragmentsBuild
buffer.append( getMultiValuedSeparator() );
index[0]++;
}
- int eo = buffer.length() < endOffset ? buffer.length() : boundaryScanner.findEndOffset( buffer, endOffset );
+ int bufferLength = buffer.length();
+ // we added the multi value char to the last buffer, ignore it
+ if (values[index[0] - 1].isTokenized()) {
+ bufferLength--;
+ }
+ int eo = bufferLength < endOffset ? bufferLength : boundaryScanner.findEndOffset( buffer, endOffset );
modifiedStartOffset[0] = boundaryScanner.findStartOffset( buffer, startOffset );
return buffer.substring( modifiedStartOffset[0], eo );
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java?rev=1232473&r1=1232472&r2=1232473&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilderTest.java Tue Jan 17 16:52:05 2012
@@ -27,7 +27,7 @@ public class ScoreOrderFragmentsBuilderT
String[] f = sofb.createFragments( reader, 0, F, ffl, 3 );
assertEquals( 3, f.length );
// check score order
- assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[0] );
+ assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
assertEquals( "<b>a</b> b b b b b b b b b b", f[2] );
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java?rev=1232473&r1=1232472&r2=1232473&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Tue Jan 17 16:52:05 2012
@@ -34,11 +34,11 @@ public class SimpleFragmentsBuilderTest
public void test1TermIndex() throws Exception {
FieldFragList ffl = ffl( "a", "a" );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
- assertEquals( "<b>a</b> ", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( "<b>a</b>", sfb.createFragment( reader, 0, F, ffl ) );
// change tags
sfb = new SimpleFragmentsBuilder( new String[]{ "[" }, new String[]{ "]" } );
- assertEquals( "[a] ", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( "[a]", sfb.createFragment( reader, 0, F, ffl ) );
}
public void test2Frags() throws Exception {
@@ -48,7 +48,7 @@ public class SimpleFragmentsBuilderTest
// 3 snippets requested, but should be 2
assertEquals( 2, f.length );
assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
- assertEquals( "b b <b>a</b> b <b>a</b> b ", f[1] );
+ assertEquals( "b b <b>a</b> b <b>a</b> b", f[1] );
}
public void test3Frags() throws Exception {
@@ -58,7 +58,7 @@ public class SimpleFragmentsBuilderTest
assertEquals( 3, f.length );
assertEquals( "<b>a</b> b b b b b b b b b b", f[0] );
assertEquals( "b b <b>a</b> b <b>a</b> b b b b b c", f[1] );
- assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b ", f[2] );
+ assertEquals( "<b>c</b> <b>a</b> <b>a</b> b b", f[2] );
}
public void testTagsAndEncoder() throws Exception {
@@ -66,7 +66,7 @@ public class SimpleFragmentsBuilderTest
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
String[] preTags = { "[" };
String[] postTags = { "]" };
- assertEquals( "<h1> [a] </h1> ",
+ assertEquals( "<h1> [a] </h1>",
sfb.createFragment( reader, 0, F, ffl, preTags, postTags, new SimpleHTMLEncoder() ) );
}
@@ -88,7 +88,7 @@ public class SimpleFragmentsBuilderTest
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
- assertEquals( "a b c <b>d</b> e ", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( "a b c <b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
}
public void test1PhraseLongMV() throws Exception {
@@ -113,7 +113,7 @@ public class SimpleFragmentsBuilderTest
SimpleFragListBuilder sflb = new SimpleFragListBuilder();
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
- assertEquals( "processing <b>speed</b>, the ", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( "processing <b>speed</b>, the", sfb.createFragment( reader, 0, F, ffl ) );
}
public void testUnstoredField() throws Exception {
@@ -163,6 +163,6 @@ public class SimpleFragmentsBuilderTest
FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
sfb.setMultiValuedSeparator( '/' );
- assertEquals( " b c//<b>d</b> e/", sfb.createFragment( reader, 0, F, ffl ) );
+ assertEquals( " b c//<b>d</b> e", sfb.createFragment( reader, 0, F, ffl ) );
}
}