You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by ko...@apache.org on 2010/10/29 18:57:12 UTC

svn commit: r1028839 - in /lucene/java/branches/lucene_3_0: ./ contrib/ contrib/analyzers/ contrib/analyzers/common/ contrib/analyzers/common/src/test/ contrib/analyzers/smartcn/ contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorh...

Author: koji
Date: Fri Oct 29 16:57:11 2010
New Revision: 1028839

URL: http://svn.apache.org/viewvc?rev=1028839&view=rev
Log:
LUCENE-2278: FastVectorHighlighter: highlighted term is out of alignment in multi-valued NOT_ANALYZED field

Modified:
    lucene/java/branches/lucene_3_0/   (props changed)
    lucene/java/branches/lucene_3_0/CHANGES.txt   (props changed)
    lucene/java/branches/lucene_3_0/build.xml   (props changed)
    lucene/java/branches/lucene_3_0/contrib/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/CHANGES.txt   (contents, props changed)
    lucene/java/branches/lucene_3_0/contrib/analyzers/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/analyzers/common/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/analyzers/common/src/test/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/analyzers/smartcn/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
    lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
    lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
    lucene/java/branches/lucene_3_0/contrib/highlighter/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/   (props changed)
    lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java   (props changed)
    lucene/java/branches/lucene_3_0/src/   (props changed)
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java   (props changed)
    lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/spans/TestSpans.java   (props changed)
    lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java   (props changed)

Propchange: lucene/java/branches/lucene_3_0/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4:748824
-/lucene/java/branches/lucene_2_9:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests:818601-821336
 /lucene/java/trunk:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/CHANGES.txt:941394,946651,948430,957490,984210,999847
 /lucene/dev/trunk/lucene/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187,989785,994979,999842
 /lucene/java/branches/lucene_2_4/CHANGES.txt:748824
-/lucene/java/branches/lucene_2_9/CHANGES.txt:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/CHANGES.txt:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/CHANGES.txt:818601-821336
 /lucene/java/trunk/CHANGES.txt:881213,881315,881466,882374,882464,882672,882807,882888,882977,883074-883075,883554,883654,883661,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/build.xml:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/build.xml:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/build.xml:748824
-/lucene/java/branches/lucene_2_9/build.xml:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/build.xml:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/build.xml:818601-821336
 /lucene/java/trunk/build.xml:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887617,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/contrib:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/contrib:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/contrib:748824
-/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/contrib:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib:818601-821336
 /lucene/java/trunk/contrib:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Modified: lucene/java/branches/lucene_3_0/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/CHANGES.txt?rev=1028839&r1=1028838&r2=1028839&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/CHANGES.txt (original)
+++ lucene/java/branches/lucene_3_0/contrib/CHANGES.txt Fri Oct 29 16:57:11 2010
@@ -10,6 +10,9 @@ Bug Fixes
  * LUCENE-2284: MatchAllDocsQueryNode toString() created an invalid XML tag.
    (Frank Wesemann via Robert Muir)
 
+ * LUCENE-2278: FastVectorHighlighter: Highlighted term is out of alignment
+   in multi-valued NOT_ANALYZED field. (Koji Sekiguchi)
+
 Documentation
 
  * LUCENE-2055: Add documentation noting that the Dutch and French stemmers

Propchange: lucene/java/branches/lucene_3_0/contrib/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/CHANGES.txt:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/contrib/CHANGES.txt:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/contrib/CHANGES.txt:748824
-/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/contrib/CHANGES.txt:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/CHANGES.txt:818601-821336
 /lucene/java/trunk/contrib/CHANGES.txt:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,919060,920270

Propchange: lucene/java/branches/lucene_3_0/contrib/analyzers/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,3 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/analyzers:941394,946651,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/contrib/analyzers:929738,935522,939611,940730,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187,989785,990281,994979,999223,999842
+/lucene/java/branches/lucene_2_9/contrib/analyzers:1028833
 /lucene/java/trunk/contrib/analyzers:887670,889579,889866,890439,891205,891377,916543,919869,920270

Propchange: lucene/java/branches/lucene_3_0/contrib/analyzers/common/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,3 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common:988629,990286,999226,999847
 /lucene/dev/trunk/lucene/contrib/analyzers/common:984187,989785,990281,994979,999223,999842
+/lucene/java/branches/lucene_2_9/contrib/analyzers/common:1028833
 /lucene/java/trunk/contrib/analyzers/common:916543,919869

Propchange: lucene/java/branches/lucene_3_0/contrib/analyzers/common/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,3 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/analyzers/common/src/test:999226,999847
 /lucene/dev/trunk/lucene/contrib/analyzers/common/src/test:989785,990281,994979,999223,999842
+/lucene/java/branches/lucene_2_9/contrib/analyzers/common/src/test:1028833
 /lucene/java/trunk/contrib/analyzers/common/src/test:916543,919869

Propchange: lucene/java/branches/lucene_3_0/contrib/analyzers/smartcn/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,3 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/analyzers/smartcn:988629,990286,999226,999847
 /lucene/dev/trunk/lucene/contrib/analyzers/smartcn:984187,989785,990281,994979,999223,999842
+/lucene/java/branches/lucene_2_9/contrib/analyzers/smartcn:1028833
 /lucene/java/trunk/contrib/analyzers/smartcn:916543,919869

Modified: lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java?rev=1028839&r1=1028838&r2=1028839&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java (original)
+++ lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Fri Oct 29 16:57:11 2010
@@ -22,6 +22,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
 import org.apache.lucene.document.MapFieldSelector;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.vectorhighlight.FieldFragList.WeightedFragInfo;
@@ -72,7 +73,7 @@ public abstract class BaseFragmentsBuild
     List<WeightedFragInfo> fragInfos = getWeightedFragInfoList( fieldFragList.fragInfos );
     
     List<String> fragments = new ArrayList<String>( maxNumFragments );
-    String[] values = getFieldValues( reader, docId, fieldName );
+    Field[] values = getFields( reader, docId, fieldName );
     if( values.length == 0 ) return null;
     StringBuilder buffer = new StringBuilder();
     int[] nextValueIndex = { 0 };
@@ -83,15 +84,31 @@ public abstract class BaseFragmentsBuild
     return fragments.toArray( new String[fragments.size()] );
   }
   
+  @Deprecated
   protected String[] getFieldValues( IndexReader reader, int docId, String fieldName) throws IOException {
     Document doc = reader.document( docId, new MapFieldSelector( new String[]{ fieldName } ) );
     return doc.getValues( fieldName ); // according to Document class javadoc, this never returns null
   }
+  
+  protected Field[] getFields( IndexReader reader, int docId, String fieldName) throws IOException {
+    // according to javadoc, doc.getFields(fieldName) cannot be used with lazy loaded field???
+    Document doc = reader.document( docId, new MapFieldSelector( new String[]{ fieldName } ) );
+    return doc.getFields( fieldName ); // according to Document class javadoc, this never returns null
+  }
 
+  @Deprecated
   protected String makeFragment( StringBuilder buffer, int[] index, String[] values, WeightedFragInfo fragInfo ){
-    StringBuilder fragment = new StringBuilder();
     final int s = fragInfo.startOffset;
-    String src = getFragmentSource( buffer, index, values, s, fragInfo.endOffset );
+    return makeFragment( fragInfo, getFragmentSource( buffer, index, values, s, fragInfo.endOffset ), s );
+  }
+
+  protected String makeFragment( StringBuilder buffer, int[] index, Field[] values, WeightedFragInfo fragInfo ){
+    final int s = fragInfo.startOffset;
+    return makeFragment( fragInfo, getFragmentSource( buffer, index, values, s, fragInfo.endOffset ), s );
+  }
+  
+  private String makeFragment( WeightedFragInfo fragInfo, String src, int s ){
+    StringBuilder fragment = new StringBuilder();
     int srcIndex = 0;
     for( SubInfo subInfo : fragInfo.subInfos ){
       for( Toffs to : subInfo.termsOffsets ){
@@ -104,6 +121,7 @@ public abstract class BaseFragmentsBuild
     return fragment.toString();
   }
   
+  @Deprecated
   protected String getFragmentSource( StringBuilder buffer, int[] index, String[] values,
       int startOffset, int endOffset ){
     while( buffer.length() < endOffset && index[0] < values.length ){
@@ -114,6 +132,17 @@ public abstract class BaseFragmentsBuild
     int eo = buffer.length() < endOffset ? buffer.length() : endOffset;
     return buffer.substring( startOffset, eo );
   }
+
+  protected String getFragmentSource( StringBuilder buffer, int[] index, Field[] values,
+      int startOffset, int endOffset ){
+    while( buffer.length() < endOffset && index[0] < values.length ){
+      if( index[0] > 0 && values[index[0]].isTokenized() && values[index[0]].stringValue().length() > 0 )
+        buffer.append( ' ' );
+      buffer.append( values[index[0]++].stringValue() );
+    }
+    int eo = buffer.length() < endOffset ? buffer.length() : endOffset;
+    return buffer.substring( startOffset, eo );
+  }
   
   protected String getPreTag( int num ){
     return preTags.length > num ? preTags[num] : preTags[0];

Modified: lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java?rev=1028839&r1=1028838&r2=1028839&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (original)
+++ lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java Fri Oct 29 16:57:11 2010
@@ -24,6 +24,7 @@ import java.util.Collection;
 import junit.framework.TestCase;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.KeywordAnalyzer;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.Tokenizer;
@@ -55,6 +56,7 @@ public abstract class AbstractTestCase e
   protected Directory dir;
   protected Analyzer analyzerW;
   protected Analyzer analyzerB;
+  protected Analyzer analyzerK;
   protected IndexReader reader;  
   protected QueryParser paW;
   protected QueryParser paB;
@@ -76,10 +78,17 @@ public abstract class AbstractTestCase e
     "\nWhen you talk about processing speed, the"
   };
 
+  protected static final String[] strMVValues = {                                                           
+    "abc",                                                                                                  
+    "defg",                                                                                                 
+    "hijkl"                                                                                                 
+  };
+
   @Override
   protected void setUp() throws Exception {
     analyzerW = new WhitespaceAnalyzer();
     analyzerB = new BigramAnalyzer();
+    analyzerK = new KeywordAnalyzer(); 
     paW = new QueryParser(Version.LUCENE_CURRENT,  F, analyzerW );
     paB = new QueryParser(Version.LUCENE_CURRENT,  F, analyzerB );
     dir = new RAMDirectory();
@@ -286,6 +295,7 @@ public abstract class AbstractTestCase e
     make1dmfIndex( analyzerB, values );
   }
   
+  // make 1 doc with multi valued field
   protected void make1dmfIndex( Analyzer analyzer, String... values ) throws Exception {
     IndexWriter writer = new IndexWriter( dir, analyzer, true, MaxFieldLength.LIMITED );
     Document doc = new Document();
@@ -297,6 +307,18 @@ public abstract class AbstractTestCase e
     reader = IndexReader.open( dir, true );
   }
   
+  // make 1 doc with multi valued & not analyzed field
+  protected void make1dmfIndexNA( String... values ) throws Exception {
+    IndexWriter writer = new IndexWriter( dir, analyzerK, true, MaxFieldLength.LIMITED );
+    Document doc = new Document();
+    for( String value: values )
+      doc.add( new Field( F, value, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
+    writer.addDocument( doc );
+    writer.close();
+
+    reader = IndexReader.open( dir, true );
+  }
+  
   protected void makeIndexShortMV() throws Exception {
 
     //  012345
@@ -358,4 +380,18 @@ public abstract class AbstractTestCase e
 
     make1dmfIndexB( biMVValues );
   }
+  
+  protected void makeIndexStrMV() throws Exception {
+
+    //  0123
+    // "abc"
+    
+    //  34567
+    // "defg"
+
+    //     111
+    //  789012
+    // "hijkl"
+    make1dmfIndexNA( strMVValues );
+  }
 }

Modified: lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java?rev=1028839&r1=1028838&r2=1028839&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java (original)
+++ lucene/java/branches/lucene_3_0/contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/SimpleFragmentsBuilderTest.java Fri Oct 29 16:57:11 2010
@@ -127,4 +127,16 @@ public class SimpleFragmentsBuilderTest 
 
     reader = IndexReader.open( dir, true );
   }
+  
+  public void test1StrMV() throws Exception {
+    makeIndexStrMV();
+
+    FieldQuery fq = new FieldQuery( tq( "defg" ), true, true );
+    FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+    FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+    SimpleFragListBuilder sflb = new SimpleFragListBuilder();
+    FieldFragList ffl = sflb.createFieldFragList( fpl, 100 );
+    SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder();
+    assertEquals( "abc<b>defg</b>hijkl", sfb.createFragment( reader, 0, F, ffl ) );
+  }
 }

Propchange: lucene/java/branches/lucene_3_0/contrib/highlighter/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,3 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/highlighter:988629,990286,999226,999847
 /lucene/dev/trunk/lucene/contrib/highlighter:984187,989785,990281,994979,999223,999842
+/lucene/java/branches/lucene_2_9/contrib/highlighter:1028833
 /lucene/java/trunk/contrib/highlighter:916543,919869

Propchange: lucene/java/branches/lucene_3_0/contrib/highlighter/src/test/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/highlighter/src/test:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/contrib/highlighter/src/test:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/contrib/highlighter/src/test:748824
-/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/contrib/highlighter/src/test:817269-818600,825998,826775,829134,829816,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/highlighter/src/test:818601-821336
 /lucene/java/trunk/contrib/highlighter/src/test:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,7 +1,7 @@
 /lucene/dev/branches/branch_3x/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:748824
-/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:817269-818600,825998,829134,829816,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:818601-821336
 /lucene/java/branches/lucene_3_0/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:889463
 /lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/src/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/src:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/src:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/src:748824
-/lucene/java/branches/lucene_2_9/src:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/src:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/src:818601-821336
 /lucene/java/trunk/src:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/Tokenizer.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/Tokenizer.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/analysis/Tokenizer.java:748824
-/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/analysis/Tokenizer.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/analysis/Tokenizer.java:818601-821336
 /lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,896661,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:748824
-/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:818601-821336
 /lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:881213,881315,881466,881984,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestDateTools.java:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestDateTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestDateTools.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestDateTools.java:825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/src/test/org/apache/lucene/document/TestNumberTools.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/document/TestNumberTools.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/document/TestNumberTools.java:825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,3 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index:941394,946651,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/src/test/org/apache/lucene/index:929738,935522,940730,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187,989785,990281,994979,999223,999842
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index:1028833
 /lucene/java/trunk/src/test/org/apache/lucene/index:887670,889579,889866,890439,891205,891377,916543,919869,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,5 +1,5 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,3 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java:999226,999847
 /lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java:989785,990281,994979,999223,999842
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java:1028833
 /lucene/java/trunk/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java:916543,919869

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,3 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java:999226,999847
 /lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java:989785,990281,994979,999223,999842
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java:1028833
 /lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java:916543,919869

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/search/spans/TestSpans.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,3 +1,4 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java:999226,999847
 /lucene/dev/trunk/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java:989785,990281,994979,999223,999842
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/search/spans/TestSpans.java:1028833
 /lucene/java/trunk/src/test/org/apache/lucene/search/spans/TestSpans.java:916543,919869

Propchange: lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Oct 29 16:57:11 2010
@@ -1,6 +1,6 @@
 /lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:941394,946651,948430,957490,984210,988629,990286,999226,999847
 /lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java:929738,932398,935522,938989,939611,939649,940730,945420,946599,948082,948429,949288,949976,949997,950458,950613,951521,953407,963781,965103,984187,989785,990281,994979,999223,999842
 /lucene/java/branches/lucene_2_4/src/test/org/apache/lucene/util/TestAttributeSource.java:748824
-/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516
+/lucene/java/branches/lucene_2_9/src/test/org/apache/lucene/util/TestAttributeSource.java:817269-818600,825998,829134,829881,831036,896850,909334,940987,948516,1028833
 /lucene/java/branches/lucene_2_9_back_compat_tests/src/test/org/apache/lucene/util/TestAttributeSource.java:818601-821336
 /lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:881213,881315,881466,881819,882374,882672,882807,882888,882977,883074-883075,883079,883554,884870,886257,886911,887347,887532,887602,887670,888247,889431-889432,889579,889866,890439,890967,890988,891189,891205,891209,891363,891377,893093,894348,897672,899627,900196,908477,908975,909360,909398,910034,910078,912407,915399,916543,919060,919869,920270