You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ro...@apache.org on 2022/02/04 17:20:51 UTC
[lucene] branch branch_9x updated (3c98bdd -> 8e5a88f)
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git.
from 3c98bdd LUCENE-10401: Fix lookups on empty doc-values terms dictionaries. (#642)
new f426ab1 LUCENE-10402: Prefix interval automaton should be declared binary (#646)
new 8e5a88f LUCENE-10407: Set bpos flag to true when containing filter is exhausted (#648)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
lucene/CHANGES.txt | 6 +++++
.../intervals/ContainingIntervalsSource.java | 4 ++-
.../apache/lucene/queries/intervals/Intervals.java | 9 ++++++-
.../queries/intervals/TestIntervalQuery.java | 29 ++++++++++++++++++----
4 files changed, 41 insertions(+), 7 deletions(-)
[lucene] 01/02: LUCENE-10402: Prefix interval automaton should be declared binary (#646)
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit f426ab13934a672236790b47ee41d69164d147a0
Author: Alan Woodward <ro...@apache.org>
AuthorDate: Fri Feb 4 15:27:03 2022 +0000
LUCENE-10402: Prefix interval automaton should be declared binary (#646)
---
lucene/CHANGES.txt | 3 +++
.../src/java/org/apache/lucene/queries/intervals/Intervals.java | 9 ++++++++-
.../org/apache/lucene/queries/intervals/TestIntervalQuery.java | 9 ++++++++-
3 files changed, 19 insertions(+), 2 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 67cf52a..beda50d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -186,6 +186,9 @@ Bug Fixes
* LUCENE-10401: Fix lookups on empty doc-value terms dictionaries to no longer
throw an ArrayIndexOutOfBoundsException. (Adrien Grand)
+* LUCENE-10402: Prefix intervals should declare their automaton as binary, otherwise prefixes
+ containing multibyte characters will not correctly match. (Alan Woodward)
+
Other
---------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java
index c2346ee..398dad0 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/Intervals.java
@@ -29,6 +29,7 @@ import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.Operations;
/**
* Factory functions for creating {@link IntervalsSource interval sources}.
@@ -157,7 +158,13 @@ public final class Intervals {
* @throws IllegalStateException if the prefix expands to more than {@code maxExpansions} terms
*/
public static IntervalsSource prefix(BytesRef prefix, int maxExpansions) {
- CompiledAutomaton ca = new CompiledAutomaton(PrefixQuery.toAutomaton(prefix));
+ CompiledAutomaton ca =
+ new CompiledAutomaton(
+ PrefixQuery.toAutomaton(prefix),
+ null,
+ true,
+ Operations.DEFAULT_DETERMINIZE_WORK_LIMIT,
+ true);
return new MultiTermIntervalsSource(ca, maxExpansions, prefix.utf8ToString() + "*");
}
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervalQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervalQuery.java
index ddd2988..190b4a1 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervalQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervalQuery.java
@@ -31,6 +31,7 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.search.CheckHits;
import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
public class TestIntervalQuery extends LuceneTestCase {
@@ -77,7 +78,8 @@ public class TestIntervalQuery extends LuceneTestCase {
"coordinate genome research",
"greater new york",
"x x x x x intend x x x message x x x message x x x addressed x x",
- "issue with intervals queries from search engine. So it's a big issue for us as we need to do ordered searches. Thank you to help us concerning that issue"
+ "issue with intervals queries from search engine. So it's a big issue for us as we need to do ordered searches. Thank you to help us concerning that issue",
+ "場外好朋友"
};
private void checkHits(Query query, int[] results) throws IOException {
@@ -416,4 +418,9 @@ public class TestIntervalQuery extends LuceneTestCase {
field, Intervals.containing(Intervals.term("w1"), new OneTimeIntervalSource()));
checkHits(q, new int[] {0, 1, 2, 3});
}
+
+ public void testUnicodePrefix() throws IOException {
+ Query q = new IntervalQuery(field, Intervals.prefix(new BytesRef("場")));
+ checkHits(q, new int[] {11});
+ }
}
[lucene] 02/02: LUCENE-10407: Set bpos flag to true when containing filter is exhausted (#648)
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
romseygeek pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 8e5a88f2d5ec68b98ff8898b42686ad7fe5216d7
Author: Alan Woodward <ro...@apache.org>
AuthorDate: Fri Feb 4 16:44:57 2022 +0000
LUCENE-10407: Set bpos flag to true when containing filter is exhausted (#648)
ContainedByIntervalIterator and OverlappingIntervalIterator set their 'is the filter
interval exhausted' flag to `false` once it has returned NO_MORE_POSITIONS on
a document, so that subsequent calls to `startPosition()` will also return
NO_MORE_POSITIONS. ContainingIntervalIterator omits to do this, and so it can
incorrectly report matches, for example when used in a disjunction. This commit
fixes that omission.
---
lucene/CHANGES.txt | 3 +++
.../queries/intervals/ContainingIntervalsSource.java | 4 +++-
.../lucene/queries/intervals/TestIntervalQuery.java | 20 ++++++++++++++++----
3 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index beda50d..4b5b670 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -189,6 +189,9 @@ Bug Fixes
* LUCENE-10402: Prefix intervals should declare their automaton as binary, otherwise prefixes
containing multibyte characters will not correctly match. (Alan Woodward)
+* LUCENE-10407: Containing intervals could sometimes yield incorrect matches when wrapped
+ in a disjunction. (Alan Woodward, Dawid Weiss)
+
Other
---------------------
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/ContainingIntervalsSource.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/ContainingIntervalsSource.java
index 900ecce..5987dc3 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/ContainingIntervalsSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/ContainingIntervalsSource.java
@@ -51,8 +51,10 @@ class ContainingIntervalsSource extends ConjunctionIntervalsSource {
}
while (a.nextInterval() != IntervalIterator.NO_MORE_INTERVALS) {
while (b.start() < a.start() && b.end() < a.end()) {
- if (b.nextInterval() == IntervalIterator.NO_MORE_INTERVALS)
+ if (b.nextInterval() == IntervalIterator.NO_MORE_INTERVALS) {
+ bpos = false;
return IntervalIterator.NO_MORE_INTERVALS;
+ }
}
if (a.start() <= b.start() && a.end() >= b.end()) {
return a.start();
diff --git a/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervalQuery.java b/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervalQuery.java
index 190b4a1..1275c1d 100644
--- a/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervalQuery.java
+++ b/lucene/queries/src/test/org/apache/lucene/queries/intervals/TestIntervalQuery.java
@@ -17,6 +17,11 @@
package org.apache.lucene.queries.intervals;
+import static org.apache.lucene.queries.intervals.Intervals.containing;
+import static org.apache.lucene.queries.intervals.Intervals.extend;
+import static org.apache.lucene.queries.intervals.Intervals.or;
+import static org.apache.lucene.queries.intervals.Intervals.term;
+
import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -50,9 +55,9 @@ public class TestIntervalQuery extends LuceneTestCase {
random(),
directory,
newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
- for (int i = 0; i < docFields.length; i++) {
+ for (String docField : docFields) {
Document doc = new Document();
- doc.add(newTextField(field, docFields[i], Field.Store.YES));
+ doc.add(newTextField(field, docField, Field.Store.YES));
writer.addDocument(doc);
}
reader = writer.getReader();
@@ -67,7 +72,7 @@ public class TestIntervalQuery extends LuceneTestCase {
super.tearDown();
}
- private String[] docFields = {
+ private final String[] docFields = {
"w1 w2 w3 w4 w5",
"w1 w3 w2 w3",
"w1 xx w2 w4 yy w3",
@@ -168,7 +173,7 @@ public class TestIntervalQuery extends LuceneTestCase {
assertTrue(explain.toString().contains(field));
}
- public void testNullConstructorArgs() throws IOException {
+ public void testNullConstructorArgs() {
expectThrows(NullPointerException.class, () -> new IntervalQuery(null, Intervals.term("z")));
expectThrows(NullPointerException.class, () -> new IntervalQuery("field", null));
}
@@ -423,4 +428,11 @@ public class TestIntervalQuery extends LuceneTestCase {
Query q = new IntervalQuery(field, Intervals.prefix(new BytesRef("場")));
checkHits(q, new int[] {11});
}
+
+ public void testExtendDisjunctions() throws IOException {
+ Query q =
+ new IntervalQuery(
+ field, or(term("XXX"), containing(extend(term("message"), 0, 10), term("intend"))));
+ checkHits(q, new int[] {});
+ }
}