You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/07/28 22:24:01 UTC

svn commit: r1693156 - in /lucene/dev/trunk/lucene: CHANGES.txt highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java

Author: mikemccand
Date: Tue Jul 28 20:24:01 2015
New Revision: 1693156

URL: http://svn.apache.org/r1693156
Log:
LUCENE-6334: fix FastVectorHighlighter when a phrase spans more than one value in a multi-valued field

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
    lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1693156&r1=1693155&r2=1693156&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Jul 28 20:24:01 2015
@@ -281,6 +281,10 @@ Bug fixes
 * LUCENE-6696: Fix FilterDirectoryReader.close() to never close the
   underlying reader several times. (Adrien Grand)
 
+* LUCENE-6334: FastVectorHighlighter failed to highlight phrases across
+  more than one value in a multi-valued field. (Chris Earle, Nik Everett
+  via Mike McCandless)
+
 Changes in Runtime Behavior
 
 * LUCENE-6501: The subreader structure in ParallelCompositeReader

Modified: lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java?rev=1693156&r1=1693155&r2=1693156&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Tue Jul 28 20:24:01 2015
@@ -268,10 +268,39 @@ public abstract class BaseFragmentsBuild
           Iterator<Toffs> toffsIterator = subInfo.getTermsOffsets().iterator();
           while (toffsIterator.hasNext()) {
             Toffs toffs = toffsIterator.next();
-            if (toffs.getStartOffset() >= fieldStart && toffs.getEndOffset() <= fieldEnd) {
-
+            if (toffs.getStartOffset() >= fieldEnd) {
+              // We've gone past this value so its not worth iterating any more.
+              break;
+            }
+            boolean startsAfterField = toffs.getStartOffset() >= fieldStart;
+            boolean endsBeforeField = toffs.getEndOffset() < fieldEnd;
+            if (startsAfterField && endsBeforeField) {
+              // The Toff is entirely within this value.
               toffsList.add(toffs);
               toffsIterator.remove();
+            } else if (startsAfterField) {
+              /*
+               * The Toffs starts within this value but ends after this value
+               * so we clamp the returned Toffs to this value and leave the
+               * Toffs in the iterator for the next value of this field.
+               */
+              toffsList.add(new Toffs(toffs.getStartOffset(), fieldEnd - 1));
+            } else if (endsBeforeField) {
+              /*
+               * The Toffs starts before this value but ends in this value
+               * which means we're really continuing from where we left off
+               * above. Since we use the remainder of the offset we can remove
+               * it from the iterator.
+               */
+              toffsList.add(new Toffs(fieldStart, toffs.getEndOffset()));
+              toffsIterator.remove();
+            } else {
+              /*
+               * The Toffs spans the whole value so we clamp on both sides.
+               * This is basically a combination of both arms of the loop
+               * above.
+               */
+              toffsList.add(new Toffs(fieldStart, fieldEnd - 1));
             }
           }
           if (!toffsList.isEmpty()) {

Modified: lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java?rev=1693156&r1=1693155&r2=1693156&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java (original)
+++ lucene/dev/trunk/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java Tue Jul 28 20:24:01 2015
@@ -574,6 +574,72 @@ public class FastVectorHighlighterTest e
     dir.close();
   }
 
+  public void testPhrasesSpanningFieldValues() throws IOException {
+    Directory dir = newDirectory();
+    // positionIncrementGap is 0 so the pharse is found across multiple field
+    // values.
+    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
+    FieldType type = new FieldType(TextField.TYPE_STORED);
+    type.setStoreTermVectorOffsets(true);
+    type.setStoreTermVectorPositions(true);
+    type.setStoreTermVectors(true);
+    type.freeze();
+
+    Document doc = new Document();
+    doc.add( new Field( "field", "one two three five", type ) );
+    doc.add( new Field( "field", "two three four", type ) );
+    doc.add( new Field( "field", "five six five", type ) );
+    doc.add( new Field( "field", "six seven eight nine eight nine eight " +
+      "nine eight nine eight nine eight nine", type ) );
+    doc.add( new Field( "field", "eight nine", type ) );
+    doc.add( new Field( "field", "ten eleven", type ) );
+    doc.add( new Field( "field", "twelve thirteen", type ) );
+    writer.addDocument(doc);
+
+    BaseFragListBuilder fragListBuilder = new SimpleFragListBuilder();
+    BaseFragmentsBuilder fragmentsBuilder = new SimpleFragmentsBuilder();
+    fragmentsBuilder.setDiscreteMultiValueHighlighting(true);
+    FastVectorHighlighter highlighter = new FastVectorHighlighter(true, true, fragListBuilder, fragmentsBuilder);
+    IndexReader reader = DirectoryReader.open(writer, true);
+    int docId = 0;
+
+    // Phrase that spans a field value
+    Query q = new PhraseQuery("field", "four", "five");
+    FieldQuery fieldQuery  = highlighter.getFieldQuery(q, reader);
+    String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 1000, 1000);
+    assertEquals("two three <b>four</b>", bestFragments[0]);
+    assertEquals("<b>five</b> six five", bestFragments[1]);
+    assertEquals(2, bestFragments.length);
+
+    // Phrase that ends at a field value
+    q = new PhraseQuery("field", "three", "five");
+    fieldQuery  = highlighter.getFieldQuery(q, reader);
+    bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 1000, 1000);
+    assertEquals("one two <b>three five</b>", bestFragments[0]);
+    assertEquals(1, bestFragments.length);
+
+    // Phrase that spans across three values
+    q = new PhraseQuery("field", "nine", "ten", "eleven", "twelve");
+    fieldQuery  = highlighter.getFieldQuery(q, reader);
+    bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 1000, 1000);
+    assertEquals("eight <b>nine</b>", bestFragments[0]);
+    assertEquals("<b>ten eleven</b>", bestFragments[1]);
+    assertEquals("<b>twelve</b> thirteen", bestFragments[2]);
+    assertEquals(3, bestFragments.length);
+
+    // Term query that appears in multiple values
+    q = new TermQuery(new Term("field", "two"));
+    fieldQuery  = highlighter.getFieldQuery(q, reader);
+    bestFragments = highlighter.getBestFragments(fieldQuery, reader, docId, "field", 1000, 1000);
+    assertEquals("one <b>two</b> three five", bestFragments[0]);
+    assertEquals("<b>two</b> three four", bestFragments[1]);
+    assertEquals(2, bestFragments.length);
+
+    reader.close();
+    writer.close();
+    dir.close();
+  }
+
   private void matchedFieldsTestCase( String fieldValue, String expected, Query... queryClauses ) throws IOException {
     matchedFieldsTestCase( true, true, fieldValue, expected, queryClauses );
   }