You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2022/04/11 10:56:57 UTC
[lucene] branch branch_9x updated: LUCENE-10229: return -1 for unknown offsets in ExtendedIntervalsSource. Modify highlighting to work properly with or without offsets (depending on their availability). (#803)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 62fe8e28747 LUCENE-10229: return -1 for unknown offsets in ExtendedIntervalsSource. Modify highlighting to work properly with or without offsets (depending on their availability). (#803)
62fe8e28747 is described below
commit 62fe8e28747f53a2ad06f4e0c6376c1de593dc63
Author: Dawid Weiss <da...@carrotsearch.com>
AuthorDate: Mon Apr 11 11:52:31 2022 +0200
LUCENE-10229: return -1 for unknown offsets in ExtendedIntervalsSource. Modify highlighting to work properly with or without offsets (depending on their availability). (#803)
Thanks @romseygeek
---
lucene/CHANGES.txt | 3 +
.../matchhighlight/MatchRegionRetriever.java | 35 +--
.../matchhighlight/OffsetsFromMatchIterator.java | 42 ++-
.../matchhighlight/OffsetsFromPositions.java | 21 +-
.../search/matchhighlight/OffsetsFromTokens.java | 5 -
.../search/matchhighlight/OffsetsFromValues.java | 5 -
.../matchhighlight/OffsetsRetrievalStrategy.java | 5 -
.../matchhighlight/TestMatchHighlighter.java | 330 ++++++++++-----------
.../intervals/ConjunctionMatchesIterator.java | 12 +-
.../queries/intervals/ExtendedIntervalsSource.java | 63 +++-
.../lucene/queries/intervals/TestIntervals.java | 57 +++-
11 files changed, 330 insertions(+), 248 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 53dcd1fe8f4..24a6685fb3c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -28,6 +28,9 @@ New Features
Improvements
---------------------
+* LUCENE-10229: return -1 for unknown offsets in ExtendedIntervalsSource. Modify highlighting to
+ work properly with or without offsets. (Dawid Weiss)
+
* LUCENE-10494: Implement method to bulk add all collection elements to a PriorityQueue.
(Bauyrzhan Sakhariyev)
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java
index 9ac3b0ca459..afd19e54083 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/MatchRegionRetriever.java
@@ -23,6 +23,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.PrimitiveIterator;
import java.util.Set;
import java.util.TreeMap;
@@ -43,6 +44,7 @@ import org.apache.lucene.search.QueryVisitor;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.IOSupplier;
/**
* Utility class to compute a list of "match regions" for a given query, searcher and document(s)
@@ -75,7 +77,7 @@ public class MatchRegionRetriever {
*/
@FunctionalInterface
public interface FieldValueProvider {
- List<CharSequence> getValues(String field);
+ List<CharSequence> getValues(String field) throws IOException;
}
/**
@@ -131,9 +133,7 @@ public class MatchRegionRetriever {
preloadFields = new HashSet<>();
offsetStrategies.forEach(
(field, strategy) -> {
- if (strategy.requiresDocument()) {
- preloadFields.add(field);
- }
+ preloadFields.add(field);
});
// Only preload those field values that can be affected by the query and are required
@@ -181,17 +181,12 @@ public class MatchRegionRetriever {
int contextRelativeDocId = docId - currentContext.docBase;
// Only preload fields we may potentially need.
- FieldValueProvider documentSupplier;
- if (preloadFields.isEmpty()) {
- documentSupplier = null;
- } else {
- Document doc = currentContext.reader().document(contextRelativeDocId, preloadFields);
- documentSupplier = new DocumentFieldValueProvider(doc);
- }
+ FieldValueProvider docFieldsSupplier =
+ new DocumentFieldValueProvider(currentContext, contextRelativeDocId, preloadFields);
highlights.clear();
highlightDocument(
- currentContext, contextRelativeDocId, documentSupplier, (field) -> true, highlights);
+ currentContext, contextRelativeDocId, docFieldsSupplier, (field) -> true, highlights);
consumer.accept(docId, currentContext.reader(), contextRelativeDocId, highlights);
}
}
@@ -262,7 +257,7 @@ public class MatchRegionRetriever {
switch (fieldInfo.getIndexOptions()) {
case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS:
- return new OffsetsFromMatchIterator(field);
+ return new OffsetsFromMatchIterator(field, new OffsetsFromPositions(field, analyzer));
case DOCS_AND_FREQS_AND_POSITIONS:
return new OffsetsFromPositions(field, analyzer);
@@ -293,14 +288,20 @@ public class MatchRegionRetriever {
/** Implements {@link FieldValueProvider} wrapping a preloaded {@link Document}. */
private static final class DocumentFieldValueProvider implements FieldValueProvider {
- private final Document doc;
+ private final IOSupplier<Document> docSupplier;
+ private Document doc;
- public DocumentFieldValueProvider(Document doc) {
- this.doc = doc;
+ public DocumentFieldValueProvider(
+ LeafReaderContext currentContext, int docId, Set<String> preloadFields) {
+ docSupplier = () -> currentContext.reader().document(docId, preloadFields);
}
@Override
- public List<CharSequence> getValues(String field) {
+ public List<CharSequence> getValues(String field) throws IOException {
+ if (doc == null) {
+ doc = Objects.requireNonNull(docSupplier.get());
+ }
+
return Arrays.asList(doc.getValues(field));
}
}
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromMatchIterator.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromMatchIterator.java
index ca270c0562b..17104a01a80 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromMatchIterator.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromMatchIterator.java
@@ -19,29 +19,53 @@ package org.apache.lucene.search.matchhighlight;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import org.apache.lucene.search.MatchesIterator;
-/** This strategy retrieves offsets directly from {@link MatchesIterator}. */
+/**
+ * This strategy retrieves offsets directly from {@link MatchesIterator}, if they are available,
+ * otherwise it falls back to using {@link OffsetsFromPositions}.
+ */
public final class OffsetsFromMatchIterator implements OffsetsRetrievalStrategy {
private final String field;
+ private final OffsetsFromPositions noOffsetsFallback;
- OffsetsFromMatchIterator(String field) {
+ OffsetsFromMatchIterator(String field, OffsetsFromPositions noOffsetsFallback) {
this.field = field;
+ this.noOffsetsFallback = Objects.requireNonNull(noOffsetsFallback);
}
@Override
public List<OffsetRange> get(
MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)
throws IOException {
- ArrayList<OffsetRange> ranges = new ArrayList<>();
+ ArrayList<OffsetRange> positionRanges = new ArrayList<>();
+ ArrayList<OffsetRange> offsetRanges = new ArrayList<>();
while (matchesIterator.next()) {
- int from = matchesIterator.startOffset();
- int to = matchesIterator.endOffset();
- if (from < 0 || to < 0) {
- throw new IOException("Matches API returned negative offsets for field: " + field);
+ int fromPosition = matchesIterator.startPosition();
+ int toPosition = matchesIterator.endPosition();
+ if (fromPosition < 0 || toPosition < 0) {
+ throw new IOException("Matches API returned negative positions for field: " + field);
+ }
+ positionRanges.add(new OffsetRange(fromPosition, toPosition));
+
+ if (offsetRanges != null) {
+ int from = matchesIterator.startOffset();
+ int to = matchesIterator.endOffset();
+ if (from < 0 || to < 0) {
+ // At least one offset isn't available. Fallback to just positions.
+ offsetRanges = null;
+ } else {
+ offsetRanges.add(new OffsetRange(from, to));
+ }
}
- ranges.add(new OffsetRange(from, to));
}
- return ranges;
+
+ // Use the fallback conversion from positions if not all offsets were available.
+ if (offsetRanges == null) {
+ return noOffsetsFallback.convertPositionsToOffsets(positionRanges, doc.getValues(field));
+ } else {
+ return offsetRanges;
+ }
}
}
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java
index e35107ca708..e460541c115 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromPositions.java
@@ -56,20 +56,11 @@ public final class OffsetsFromPositions implements OffsetsRetrievalStrategy {
}
// Convert from positions to offsets.
- return convertPositionsToOffsets(positionRanges, analyzer, field, doc.getValues(field));
+ return convertPositionsToOffsets(positionRanges, doc.getValues(field));
}
- @Override
- public boolean requiresDocument() {
- return true;
- }
-
- private static List<OffsetRange> convertPositionsToOffsets(
- ArrayList<OffsetRange> positionRanges,
- Analyzer analyzer,
- String fieldName,
- List<CharSequence> values)
- throws IOException {
+ List<OffsetRange> convertPositionsToOffsets(
+ ArrayList<OffsetRange> positionRanges, List<CharSequence> values) throws IOException {
if (positionRanges.isEmpty()) {
return positionRanges;
@@ -106,7 +97,7 @@ public final class OffsetsFromPositions implements OffsetsRetrievalStrategy {
final String value = values.get(valueIndex).toString();
final boolean lastValue = valueIndex + 1 == max;
- TokenStream ts = analyzer.tokenStream(fieldName, value);
+ TokenStream ts = analyzer.tokenStream(field, value);
OffsetAttribute offsetAttr = ts.getAttribute(OffsetAttribute.class);
PositionIncrementAttribute posAttr = ts.getAttribute(PositionIncrementAttribute.class);
ts.reset();
@@ -144,8 +135,8 @@ public final class OffsetsFromPositions implements OffsetsRetrievalStrategy {
}
}
ts.end();
- position += posAttr.getPositionIncrement() + analyzer.getPositionIncrementGap(fieldName);
- valueOffset += offsetAttr.endOffset() + analyzer.getOffsetGap(fieldName);
+ position += posAttr.getPositionIncrement() + analyzer.getPositionIncrementGap(field);
+ valueOffset += offsetAttr.endOffset() + analyzer.getOffsetGap(field);
ts.close();
}
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java
index 913d89c7a1a..3861c4e8cc0 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromTokens.java
@@ -91,9 +91,4 @@ public final class OffsetsFromTokens implements OffsetsRetrievalStrategy {
}
return ranges;
}
-
- @Override
- public boolean requiresDocument() {
- return true;
- }
}
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java
index 108fed577e9..edb68936260 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsFromValues.java
@@ -66,9 +66,4 @@ public final class OffsetsFromValues implements OffsetsRetrievalStrategy {
}
return ranges;
}
-
- @Override
- public boolean requiresDocument() {
- return true;
- }
}
diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategy.java
index 3cbd3d412c0..9fdf0d58332 100644
--- a/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategy.java
+++ b/lucene/highlighter/src/java/org/apache/lucene/search/matchhighlight/OffsetsRetrievalStrategy.java
@@ -30,9 +30,4 @@ public interface OffsetsRetrievalStrategy {
List<OffsetRange> get(
MatchesIterator matchesIterator, MatchRegionRetriever.FieldValueProvider doc)
throws IOException;
-
- /** Whether this strategy requires document field access. */
- default boolean requiresDocument() {
- return false;
- }
}
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java
index 5babc1c134b..00ca8d7b1c1 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/matchhighlight/TestMatchHighlighter.java
@@ -337,179 +337,163 @@ public class TestMatchHighlighter extends LuceneTestCase {
String inputDocument = "The quick brown fox jumps over the lazy dog";
List<String[]> queryResultPairs =
- new ArrayList<>(
- Arrays.asList(
- new String[][] {
- {
- "fn:ordered(brown dog)",
- "0. %s: The quick >brown fox jumps over the lazy dog<"
- },
- {
- "fn:within(fn:or(lazy quick) 1 fn:or(dog fox))",
- "0. %s: The quick brown fox jumps over the >lazy< dog"
- },
- {
- "fn:containedBy(fox fn:ordered(brown fox dog))",
- "0. %s: The quick brown >fox< jumps over the lazy dog"
- },
- {
- "fn:atLeast(2 quick fox \"furry dog\")",
- "0. %s: The >quick brown fox< jumps over the lazy dog"
- },
- {
- "fn:maxgaps(0 fn:ordered(fn:or(quick lazy) fn:or(fox dog)))",
- "0. %s: The quick brown fox jumps over the >lazy dog<"
- },
- {
- "fn:maxgaps(1 fn:ordered(fn:or(quick lazy) fn:or(fox dog)))",
- "0. %s: The >quick brown fox< jumps over the >lazy dog<"
- },
- {
- "fn:maxwidth(2 fn:ordered(fn:or(quick lazy) fn:or(fox dog)))",
- "0. %s: The quick brown fox jumps over the >lazy dog<"
- },
- {
- "fn:maxwidth(3 fn:ordered(fn:or(quick lazy) fn:or(fox dog)))",
- "0. %s: The >quick brown fox< jumps over the >lazy dog<"
- },
- {
- "fn:or(quick \"fox\")",
- "0. %s: The >quick< brown >fox< jumps over the lazy dog"
- },
- {"fn:or(\"quick fox\")"},
- {
- "fn:phrase(quick brown fox)",
- "0. %s: The >quick brown fox< jumps over the lazy dog"
- },
- {"fn:wildcard(jump*)", "0. %s: The quick brown fox >jumps< over the lazy dog"},
- {"fn:wildcard(br*n)", "0. %s: The quick >brown< fox jumps over the lazy dog"},
- {"fn:fuzzyTerm(fxo)", "0. %s: The quick brown >fox< jumps over the lazy dog"},
- {"fn:or(dog fox)", "0. %s: The quick brown >fox< jumps over the lazy >dog<"},
- {
- "fn:phrase(fn:ordered(quick fox) jumps)",
- "0. %s: The >quick brown fox jumps< over the lazy dog"
- },
- {
- "fn:ordered(quick jumps dog)",
- "0. %s: The >quick brown fox jumps over the lazy dog<"
- },
- {
- "fn:ordered(quick fn:or(fox dog))",
- "0. %s: The >quick brown fox< jumps over the lazy dog"
- },
- {
- "fn:ordered(quick jumps fn:or(fox dog))",
- "0. %s: The >quick brown fox jumps over the lazy dog<"
- },
- {
- "fn:unordered(dog jumps quick)",
- "0. %s: The >quick brown fox jumps over the lazy dog<"
- },
- {
- "fn:unordered(fn:or(fox dog) quick)",
- "0. %s: The >quick brown fox< jumps over the lazy dog"
- },
- {
- "fn:unordered(fn:phrase(brown fox) fn:phrase(fox jumps))",
- "0. %s: The quick >brown fox jumps< over the lazy dog"
- },
- {"fn:ordered(fn:phrase(brown fox) fn:phrase(fox jumps))"},
- {"fn:unorderedNoOverlaps(fn:phrase(brown fox) fn:phrase(fox jumps))"},
- {
- "fn:before(fn:or(brown lazy) fox)",
- "0. %s: The quick >brown< fox jumps over the lazy dog"
- },
- {
- "fn:before(fn:or(brown lazy) fn:or(dog fox))",
- "0. %s: The quick >brown< fox jumps over the >lazy< dog"
- },
- {
- "fn:after(fn:or(brown lazy) fox)",
- "0. %s: The quick brown fox jumps over the >lazy< dog"
- },
- {
- "fn:after(fn:or(brown lazy) fn:or(dog fox))",
- "0. %s: The quick brown fox jumps over the >lazy< dog"
- },
- {
- "fn:within(fn:or(fox dog) 1 fn:or(quick lazy))",
- "0. %s: The quick brown fox jumps over the lazy >dog<"
- },
- {
- "fn:within(fn:or(fox dog) 2 fn:or(quick lazy))",
- "0. %s: The quick brown >fox< jumps over the lazy >dog<"
- },
- {
- "fn:notWithin(fn:or(fox dog) 1 fn:or(quick lazy))",
- "0. %s: The quick brown >fox< jumps over the lazy dog"
- },
- {
- "fn:containedBy(fn:or(fox dog) fn:ordered(quick lazy))",
- "0. %s: The quick brown >fox< jumps over the lazy dog"
- },
- {
- "fn:notContainedBy(fn:or(fox dog) fn:ordered(quick lazy))",
- "0. %s: The quick brown fox jumps over the lazy >dog<"
- },
- {
- "fn:containing(fn:atLeast(2 quick fox dog) jumps)",
- "0. %s: The quick brown >fox jumps over the lazy dog<"
- },
- {
- "fn:notContaining(fn:ordered(fn:or(the The) fn:or(fox dog)) brown)",
- "0. %s: The quick brown fox jumps over >the lazy dog<"
- },
- {
- "fn:overlapping(fn:phrase(brown fox) fn:phrase(fox jumps))",
- "0. %s: The quick >brown fox< jumps over the lazy dog"
- },
- {
- "fn:overlapping(fn:or(fox dog) fn:extend(lazy 2 2))",
- "0. %s: The quick brown fox jumps over the lazy >dog<"
- },
- {
- "fn:nonOverlapping(fn:phrase(brown fox) fn:phrase(lazy dog))",
- "0. %s: The quick >brown fox< jumps over the lazy dog"
- },
- {
- "fn:nonOverlapping(fn:or(fox dog) fn:extend(lazy 2 2))",
- "0. %s: The quick brown >fox< jumps over the lazy dog"
- },
- {
- "fn:atLeast(2 fn:unordered(furry dog) fn:unordered(brown dog) lazy quick)",
- "0. %s: The >quick >brown fox jumps over the lazy<<> dog<"
- },
- }));
-
- // TODO: LUCENE-10229: The test cases below do not work for fields enabled with offset yet:
- // mainly "extend".
- if (field.equals(FLD_TEXT2)) {
- queryResultPairs.addAll(
- Arrays.asList(
- new String[][] {
- {"fn:extend(fox 1 2)", "0. %s: The quick >brown fox jumps over< the lazy dog"},
- {
- "fn:extend(fn:or(dog fox) 2 0)",
- "0. %s: The >quick brown fox< jumps over >the lazy dog<"
- },
- {
- "fn:containedBy(fn:or(fox dog) fn:extend(lazy 3 3))",
- "0. %s: The quick brown fox jumps over the lazy >dog<"
- },
- {
- "fn:notContainedBy(fn:or(fox dog) fn:extend(lazy 3 3))",
- "0. %s: The quick brown >fox< jumps over the lazy dog"
- },
- {
- "fn:containing(fn:extend(fn:or(lazy brown) 1 1) fn:or(fox dog))",
- "0. %s: The >quick brown fox< jumps over >the lazy dog<"
- },
- {
- "fn:notContaining(fn:extend(fn:or(fox dog) 1 0) fn:or(brown yellow))",
- "0. %s: The quick brown fox jumps over the >lazy dog<"
- }
- }));
- }
+ Arrays.asList(
+ new String[][] {
+ {"fn:ordered(brown dog)", "0. %s: The quick >brown fox jumps over the lazy dog<"},
+ {
+ "fn:within(fn:or(lazy quick) 1 fn:or(dog fox))",
+ "0. %s: The quick brown fox jumps over the >lazy< dog"
+ },
+ {
+ "fn:containedBy(fox fn:ordered(brown fox dog))",
+ "0. %s: The quick brown >fox< jumps over the lazy dog"
+ },
+ {
+ "fn:atLeast(2 quick fox \"furry dog\")",
+ "0. %s: The >quick brown fox< jumps over the lazy dog"
+ },
+ {
+ "fn:maxgaps(0 fn:ordered(fn:or(quick lazy) fn:or(fox dog)))",
+ "0. %s: The quick brown fox jumps over the >lazy dog<"
+ },
+ {
+ "fn:maxgaps(1 fn:ordered(fn:or(quick lazy) fn:or(fox dog)))",
+ "0. %s: The >quick brown fox< jumps over the >lazy dog<"
+ },
+ {
+ "fn:maxwidth(2 fn:ordered(fn:or(quick lazy) fn:or(fox dog)))",
+ "0. %s: The quick brown fox jumps over the >lazy dog<"
+ },
+ {
+ "fn:maxwidth(3 fn:ordered(fn:or(quick lazy) fn:or(fox dog)))",
+ "0. %s: The >quick brown fox< jumps over the >lazy dog<"
+ },
+ {"fn:or(quick \"fox\")", "0. %s: The >quick< brown >fox< jumps over the lazy dog"},
+ {"fn:or(\"quick fox\")"},
+ {
+ "fn:phrase(quick brown fox)",
+ "0. %s: The >quick brown fox< jumps over the lazy dog"
+ },
+ {"fn:wildcard(jump*)", "0. %s: The quick brown fox >jumps< over the lazy dog"},
+ {"fn:wildcard(br*n)", "0. %s: The quick >brown< fox jumps over the lazy dog"},
+ {"fn:fuzzyTerm(fxo)", "0. %s: The quick brown >fox< jumps over the lazy dog"},
+ {"fn:or(dog fox)", "0. %s: The quick brown >fox< jumps over the lazy >dog<"},
+ {
+ "fn:phrase(fn:ordered(quick fox) jumps)",
+ "0. %s: The >quick brown fox jumps< over the lazy dog"
+ },
+ {
+ "fn:ordered(quick jumps dog)",
+ "0. %s: The >quick brown fox jumps over the lazy dog<"
+ },
+ {
+ "fn:ordered(quick fn:or(fox dog))",
+ "0. %s: The >quick brown fox< jumps over the lazy dog"
+ },
+ {
+ "fn:ordered(quick jumps fn:or(fox dog))",
+ "0. %s: The >quick brown fox jumps over the lazy dog<"
+ },
+ {
+ "fn:unordered(dog jumps quick)",
+ "0. %s: The >quick brown fox jumps over the lazy dog<"
+ },
+ {
+ "fn:unordered(fn:or(fox dog) quick)",
+ "0. %s: The >quick brown fox< jumps over the lazy dog"
+ },
+ {
+ "fn:unordered(fn:phrase(brown fox) fn:phrase(fox jumps))",
+ "0. %s: The quick >brown fox jumps< over the lazy dog"
+ },
+ {"fn:ordered(fn:phrase(brown fox) fn:phrase(fox jumps))"},
+ {"fn:unorderedNoOverlaps(fn:phrase(brown fox) fn:phrase(fox jumps))"},
+ {
+ "fn:before(fn:or(brown lazy) fox)",
+ "0. %s: The quick >brown< fox jumps over the lazy dog"
+ },
+ {
+ "fn:before(fn:or(brown lazy) fn:or(dog fox))",
+ "0. %s: The quick >brown< fox jumps over the >lazy< dog"
+ },
+ {
+ "fn:after(fn:or(brown lazy) fox)",
+ "0. %s: The quick brown fox jumps over the >lazy< dog"
+ },
+ {
+ "fn:after(fn:or(brown lazy) fn:or(dog fox))",
+ "0. %s: The quick brown fox jumps over the >lazy< dog"
+ },
+ {
+ "fn:within(fn:or(fox dog) 1 fn:or(quick lazy))",
+ "0. %s: The quick brown fox jumps over the lazy >dog<"
+ },
+ {
+ "fn:within(fn:or(fox dog) 2 fn:or(quick lazy))",
+ "0. %s: The quick brown >fox< jumps over the lazy >dog<"
+ },
+ {
+ "fn:notWithin(fn:or(fox dog) 1 fn:or(quick lazy))",
+ "0. %s: The quick brown >fox< jumps over the lazy dog"
+ },
+ {
+ "fn:containedBy(fn:or(fox dog) fn:ordered(quick lazy))",
+ "0. %s: The quick brown >fox< jumps over the lazy dog"
+ },
+ {
+ "fn:notContainedBy(fn:or(fox dog) fn:ordered(quick lazy))",
+ "0. %s: The quick brown fox jumps over the lazy >dog<"
+ },
+ {
+ "fn:containing(fn:atLeast(2 quick fox dog) jumps)",
+ "0. %s: The quick brown >fox jumps over the lazy dog<"
+ },
+ {
+ "fn:notContaining(fn:ordered(fn:or(the The) fn:or(fox dog)) brown)",
+ "0. %s: The quick brown fox jumps over >the lazy dog<"
+ },
+ {
+ "fn:overlapping(fn:phrase(brown fox) fn:phrase(fox jumps))",
+ "0. %s: The quick >brown fox< jumps over the lazy dog"
+ },
+ {
+ "fn:overlapping(fn:or(fox dog) fn:extend(lazy 2 2))",
+ "0. %s: The quick brown fox jumps over the lazy >dog<"
+ },
+ {
+ "fn:nonOverlapping(fn:phrase(brown fox) fn:phrase(lazy dog))",
+ "0. %s: The quick >brown fox< jumps over the lazy dog"
+ },
+ {
+ "fn:nonOverlapping(fn:or(fox dog) fn:extend(lazy 2 2))",
+ "0. %s: The quick brown >fox< jumps over the lazy dog"
+ },
+ {
+ "fn:atLeast(2 fn:unordered(furry dog) fn:unordered(brown dog) lazy quick)",
+ "0. %s: The >quick >brown fox jumps over the lazy<<> dog<"
+ },
+ {"fn:extend(fox 1 2)", "0. %s: The quick >brown fox jumps over< the lazy dog"},
+ {
+ "fn:extend(fn:or(dog fox) 2 0)",
+ "0. %s: The >quick brown fox< jumps over >the lazy dog<"
+ },
+ {
+ "fn:containedBy(fn:or(fox dog) fn:extend(lazy 3 3))",
+ "0. %s: The quick brown fox jumps over the lazy >dog<"
+ },
+ {
+ "fn:notContainedBy(fn:or(fox dog) fn:extend(lazy 3 3))",
+ "0. %s: The quick brown >fox< jumps over the lazy dog"
+ },
+ {
+ "fn:containing(fn:extend(fn:or(lazy brown) 1 1) fn:or(fox dog))",
+ "0. %s: The >quick brown fox< jumps over >the lazy dog<"
+ },
+ {
+ "fn:notContaining(fn:extend(fn:or(fox dog) 1 0) fn:or(brown yellow))",
+ "0. %s: The quick brown fox jumps over the >lazy dog<"
+ }
+ });
// Verify assertions.
new IndexBuilder(this::toField)
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/ConjunctionMatchesIterator.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/ConjunctionMatchesIterator.java
index 311fdd4ace6..ef74ec2e13b 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/ConjunctionMatchesIterator.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/ConjunctionMatchesIterator.java
@@ -60,7 +60,11 @@ class ConjunctionMatchesIterator implements IntervalMatchesIterator {
public int startOffset() throws IOException {
int start = Integer.MAX_VALUE;
for (MatchesIterator s : subs) {
- start = Math.min(start, s.startOffset());
+ int v = s.startOffset();
+ if (v == -1) {
+ return -1;
+ }
+ start = Math.min(start, v);
}
return start;
}
@@ -69,7 +73,11 @@ class ConjunctionMatchesIterator implements IntervalMatchesIterator {
public int endOffset() throws IOException {
int end = -1;
for (MatchesIterator s : subs) {
- end = Math.max(end, s.endOffset());
+ int v = s.endOffset();
+ if (v == -1) {
+ return -1;
+ }
+ end = Math.max(end, v);
}
return end;
}
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/ExtendedIntervalsSource.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/ExtendedIntervalsSource.java
index 45a158e6c6f..5cf028c772c 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/ExtendedIntervalsSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/ExtendedIntervalsSource.java
@@ -23,6 +23,8 @@ import java.util.Collections;
import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.MatchesIterator;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.QueryVisitor;
class ExtendedIntervalsSource extends IntervalsSource {
@@ -53,9 +55,66 @@ class ExtendedIntervalsSource extends IntervalsSource {
if (in == null) {
return null;
}
+
+ IntervalMatchesIterator inNoOffsets =
+ new IntervalMatchesIterator() {
+ IntervalMatchesIterator delegate = in;
+
+ @Override
+ public int gaps() {
+ return delegate.gaps();
+ }
+
+ @Override
+ public int width() {
+ return delegate.width();
+ }
+
+ @Override
+ public boolean next() throws IOException {
+ return delegate.next();
+ }
+
+ @Override
+ public int startPosition() {
+ return delegate.startPosition();
+ }
+
+ @Override
+ public int endPosition() {
+ return delegate.endPosition();
+ }
+
+ @Override
+ public int startOffset() throws IOException {
+ // We could return this:
+ // before > 0 ? -1 : in.startOffset();
+ // but keep it consistent for start/end offset:
+ return -1;
+ }
+
+ @Override
+ public int endOffset() throws IOException {
+ // We could return this:
+ // after > 0 ? -1 : in.startOffset();
+ // but keep it consistent for start/end offset:
+ return -1;
+ }
+
+ @Override
+ public MatchesIterator getSubMatches() throws IOException {
+ return delegate.getSubMatches();
+ }
+
+ @Override
+ public Query getQuery() {
+ return delegate.getQuery();
+ }
+ };
+
IntervalIterator wrapped =
- new ExtendedIntervalIterator(IntervalMatches.wrapMatches(in, doc), before, after);
- return IntervalMatches.asMatches(wrapped, in, doc);
+ new ExtendedIntervalIterator(IntervalMatches.wrapMatches(inNoOffsets, doc), before, after);
+ return IntervalMatches.asMatches(wrapped, inNoOffsets, doc);
}
@Override
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervals.java b/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervals.java
index 15d3a480c6f..d5d520ac22c 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervals.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervals.java
@@ -67,7 +67,8 @@ public class TestIntervals extends LuceneTestCase {
"Pease porridge cold, pease porridge hot, pease porridge in the pot twelve days old. Some like it cold, some like it hot, some like it in the fraggle",
"Nor here, nowt hot going on in pease this one",
"Pease porridge hot, pease porridge cold, pease porridge in the pot nine years old. Some like it hot, some like it twelve",
- "Porridge is great"
+ "Porridge is great",
+ ""
};
// 0 1 2 3 4 5 6 7 8 9
@@ -79,7 +80,8 @@ public class TestIntervals extends LuceneTestCase {
"a b a c b a b c",
"So thrice five miles of fertile ground",
"Pease hot porridge porridge",
- "w1 w2 w3 w4 w1 w6 w3 w8 w4 w7 w1 w6"
+ "w1 w2 w3 w4 w1 w6 w3 w8 w4 w7 w1 w6",
+ "the quick brown fox jumps over the lazy dog"
};
private static Directory directory;
@@ -117,8 +119,13 @@ public class TestIntervals extends LuceneTestCase {
IOUtils.close(searcher.getIndexReader(), directory);
}
+ /**
+ * @param expectedIntervals An array of expected position intervals for each document. Each
+ * sub-array contains a number of pairs corresponding to (start, end) positions of each
+ * interval.
+ */
private void checkIntervals(
- IntervalsSource source, String field, int expectedMatchCount, int[][] expected)
+ IntervalsSource source, String field, int expectedMatchCount, int[][] expectedIntervals)
throws IOException {
int matchedDocs = 0;
for (LeafReaderContext ctx : searcher.getIndexReader().leaves()) {
@@ -138,13 +145,17 @@ public class TestIntervals extends LuceneTestCase {
assertEquals(-1, intervals.start());
assertEquals(-1, intervals.end());
while ((pos = intervals.nextInterval()) != IntervalIterator.NO_MORE_INTERVALS) {
- if (i >= expected[id].length) {
+ if (i >= expectedIntervals[id].length) {
fail("Unexpected match in doc " + id + ": " + intervals);
}
- assertEquals(source + ": wrong start value in doc " + id, expected[id][i], pos);
assertEquals(
- "start() != pos returned from nextInterval()", expected[id][i], intervals.start());
- assertEquals("Wrong end value in doc " + id, expected[id][i + 1], intervals.end());
+ source + ": wrong start value in doc " + id, expectedIntervals[id][i], pos);
+ assertEquals(
+ "start() != pos returned from nextInterval()",
+ expectedIntervals[id][i],
+ intervals.start());
+ assertEquals(
+ "Wrong end value in doc " + id, expectedIntervals[id][i + 1], intervals.end());
i += 2;
assertTrue(mi.next());
assertEquals(
@@ -156,7 +167,8 @@ public class TestIntervals extends LuceneTestCase {
intervals.end(),
mi.endPosition());
}
- assertEquals(source + ": wrong number of endpoints in doc " + id, expected[id].length, i);
+ assertEquals(
+ source + ": wrong number of endpoints in doc " + id, expectedIntervals[id].length, i);
assertEquals(IntervalIterator.NO_MORE_INTERVALS, intervals.start());
assertEquals(IntervalIterator.NO_MORE_INTERVALS, intervals.end());
if (i > 0) {
@@ -166,7 +178,11 @@ public class TestIntervals extends LuceneTestCase {
assertNull("Expected null matches iterator on doc " + id, mi);
}
} else {
- assertEquals(0, expected[id].length);
+ // Assume empty matches if the expected array of IDs is shorter
+ // than the number of docs.
+ if (expectedIntervals.length > id) {
+ assertEquals(0, expectedIntervals[id].length);
+ }
assertNull(mi);
}
}
@@ -218,10 +234,10 @@ public class TestIntervals extends LuceneTestCase {
private void assertMatch(MatchesIterator mi, int start, int end, int startOffset, int endOffset)
throws IOException {
assertTrue(mi.next());
- assertEquals(start, mi.startPosition());
- assertEquals(end, mi.endPosition());
- assertEquals(startOffset, mi.startOffset());
- assertEquals(endOffset, mi.endOffset());
+ assertEquals("Start position", start, mi.startPosition());
+ assertEquals("End position", end, mi.endPosition());
+ assertEquals("Start offset", startOffset, mi.startOffset());
+ assertEquals("End offset", endOffset, mi.endOffset());
}
private void assertGaps(IntervalsSource source, int doc, String field, int[] expectedGaps)
@@ -251,6 +267,17 @@ public class TestIntervals extends LuceneTestCase {
e.getMessage());
}
+ public void testExtends() throws IOException {
+ IntervalsSource source = Intervals.extend(Intervals.term("fox"), 1, 2);
+ checkIntervals(source, "field2", 1, new int[][] {{}, {}, {}, {}, {}, {}, {2, 5}});
+ MatchesIterator mi = getMatches(source, 6, "field2");
+
+ // LUCENE-10229: we can't report offsets for the "extended" position range because this
+ // information
+ // is not available from term positions index alone. Report the truth (-1 - not available).
+ assertMatch(mi, 2, 5, -1, -1);
+ }
+
public void testTermQueryIntervals() throws IOException {
IntervalsSource source = Intervals.term("porridge");
checkIntervals(
@@ -838,11 +865,11 @@ public class TestIntervals extends LuceneTestCase {
assertEquals(5, source.minExtent());
MatchesIterator mi = getMatches(source, 1, "field1");
- assertMatch(mi, 3, 7, 20, 55);
+ assertMatch(mi, 3, 7, -1, -1);
MatchesIterator sub = mi.getSubMatches();
assertNotNull(sub);
assertMatch(sub, 3, 3, 20, 25);
- assertMatch(sub, 4, 6, 35, 39);
+ assertMatch(sub, 4, 6, -1, -1);
assertMatch(sub, 7, 7, 47, 55);
source = Intervals.extend(Intervals.term("w1"), 5, Integer.MAX_VALUE);